eva-sub-cli 0.4.6__tar.gz → 0.4.8__tar.gz

eva_sub_cli-0.4.8/.github/workflows/trigger-eva-sub-cli-integration-tests.yml

@@ -0,0 +1,16 @@
+name: Trigger Eva Sub Cli Integration Tests
+
+on:
+  push:
+    branches: [main, test-trigger-integration-tests]
+
+jobs:
+  trigger-integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger eva sub cli integration tests
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          token: ${{ secrets.TRIGGER_INTEGRATION_TESTS }}
+          repository: EBIvariation/eva-integration-tests
+          event-type: trigger-eva-sub-cli-tests

{eva_sub_cli-0.4.6/eva_sub_cli.egg-info → eva_sub_cli-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.4.6
+Version: 0.4.8
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -19,12 +19,14 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ebi_eva_common_pyutils==0.*,>0.6.11
+Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.1
 Requires-Dist: jinja2==3.*,>=3.0.0
 Requires-Dist: jsonschema==4.*
 Requires-Dist: openpyxl==3.*
 Requires-Dist: pyyaml==6.*
 Requires-Dist: requests==2.*
+Requires-Dist: packaging==24.*
+Requires-Dist: setuptools-scm
 Dynamic: license-file
 
 # eva-sub-cli

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.8}/eva_sub_cli/__init__.py

@@ -6,7 +6,7 @@ PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__))
 NEXTFLOW_DIR = os.path.join(PACKAGE_DIR, 'nextflow')
 ETC_DIR = os.path.join(PACKAGE_DIR, 'etc')
 LSRI_CLIENT_ID = "aa0fcc42-096a-4f9d-b871-aceb1a97d174"
-DEFAULT_METADATA_XLSX_TEMPLATE_VERSION = '1.1.6'
+MINIMUM_METADATA_XLSX_TEMPLATE_VERSION = '2.0.0'
 
 SUB_CLI_CONFIG_FILE = ".eva_sub_cli_config.yml"
 

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.8}/eva_sub_cli/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.6'
-__version_tuple__ = version_tuple = (0, 4, 6)
+__version__ = version = '0.4.8'
+__version_tuple__ = version_tuple = (0, 4, 8)

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.8}/eva_sub_cli/etc/eva_schema.json

@@ -75,11 +75,13 @@
       "properties": {
         "title": {
           "type": "string",
-          "description": "Title of the project"
+          "description": "Title of the project",
+          "maxLength": 500
         },
         "description": {
           "type": "string",
-          "description": "Description of the project"
+          "description": "Description of the project",
+          "maxLength": 5000
         },
         "centre": {
           "type": "string",
@@ -166,11 +168,11 @@
             "type": "string",
             "description": "Reference against which the analysis was performed. ENA or NCBI accession (starting with GCA) accepted."
           },
-           "referenceFasta": {
+          "referenceFasta": {
             "type": "string",
             "description": "The path to the reference in fasta format that was used to create the VCF. (This is a local path)"
           },
-           "assemblyReport": {
+          "assemblyReport": {
             "type": "string",
             "description": "The path to the assembly report associated with the assembly as found in NCBI assemblies. (This is a local path and only applicable for GCAs)"
           },
@@ -308,9 +310,10 @@
             "type": "string",
             "description": "Sample Name used in the VCF file"
           },
-          "sampleAccession": {
+          "bioSampleAccession": {
             "type": "string",
-            "description": "Accession of the sample (BioSamples, ENA or EGA)"
+            "pattern": "^SAM(E|D|N)[A-Z]?[0-9]+$",
+            "description": "BioSample accession of pre-existing sample. Only use for preregistered samples"
           },
           "bioSampleObject": {
             "type": "object",
@@ -319,7 +322,7 @@
                 "$ref": "eva-biosamples.json"
               },
               {
-                 "$ref": "https://www.ebi.ac.uk/biosamples/schema/store/registry/schemas/ERC000011"
+                "$ref": "https://www.ebi.ac.uk/biosamples/schema/store/registry/schemas/ERC000011"
               },
               {
                 "$ref": "https://www.ebi.ac.uk/biosamples/schemas/certification/plant-miappe.json"

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.8}/eva_sub_cli/executables/check_fasta_insdc.py

@@ -1,8 +1,5 @@
 import argparse
-import gzip
 import hashlib
-import json
-from itertools import groupby
 
 import requests
 import yaml
@@ -11,8 +8,7 @@ from requests import HTTPError
 from retry import retry
 
 from eva_sub_cli.file_utils import fasta_iter
-from eva_sub_cli.metadata_utils import get_files_per_analysis, get_analysis_for_vcf_file, \
-    get_reference_assembly_for_analysis
+from eva_sub_cli.metadata import EvaMetadataJson
 
 REFGET_SERVER = 'https://www.ebi.ac.uk/ena/cram'
 CONTIG_ALIAS_SERVER = 'https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum'
@@ -52,7 +48,7 @@ def _get_containing_assemblies_paged(url):
                 results.add(contigEntity['assembly']['insdcAccession'])
         if '_links' in response_data and 'next' in response_data['_links']:
             # Add results from next page if needed
-            results |= _get_containing_assemblies_paged(response_data['_links']['next'])
+            results = results.union(_get_containing_assemblies_paged(response_data['_links']['next']['href']))
         return results
     return set()
 
@@ -94,7 +90,7 @@ def assess_fasta(input_fasta, analyses, assembly_in_metadata):
                 possible_assemblies = containing_assemblies
             else:
                 possible_assemblies &= containing_assemblies
-    except (ConnectionError, HTTPError) as e:
+    except Exception as e:
         # Server errors from either ENA refget or EVA contig alias will halt the check prematurely.
         # Report the error but do not return from the method, so that incomplete results can be reported
         # (i.e. any sequences found to be INSDC and any compatible assemblies so far)
@@ -121,23 +117,21 @@ def get_analyses_and_reference_genome_from_metadata(vcf_files_for_fasta, json_fi
       single assembly accession
     :param json_file: JSON file of the metadata
     """
-    with open(json_file) as open_json:
-        metadata = json.load(open_json)
-        files_per_analysis = get_files_per_analysis(metadata)
-        # Get all analyses associated with all vcf files that are linked with a single fasta file
-        all_analyses = set()
-        for vcf_file in vcf_files_for_fasta:
-            analysis_aliases = get_analysis_for_vcf_file(vcf_file, files_per_analysis)
-            if len(analysis_aliases) != 1:
-                logger.error(f'Could not determine analysis associated with VCF file: {vcf_file}')
-            else:
-                all_analyses.add(analysis_aliases[0])
-        # Get (single) assembly associated with these analyses
-        assemblies = [get_reference_assembly_for_analysis(metadata, analysis) for analysis in all_analyses]
-        if len(assemblies) != 1:
-            logger.error(f'Could not determine assembly accession to check against fasta file, out of: {assemblies}')
-            return all_analyses, None
-        return all_analyses, assemblies[0]
+    metadata = EvaMetadataJson(json_file)
+    # Get all analyses associated with all vcf files that are linked with a single fasta file
+    all_analyses = set()
+    for vcf_file in vcf_files_for_fasta:
+        analysis_aliases = metadata.get_analysis_for_vcf_file(vcf_file)
+        if len(analysis_aliases) != 1:
+            logger.error(f'Could not determine analysis associated with VCF file: {vcf_file}')
+        else:
+            all_analyses.add(analysis_aliases[0])
+    # Get (single) assembly associated with these analyses
+    assemblies = [metadata.get_reference_assembly_for_analysis(analysis) for analysis in all_analyses]
+    if len(assemblies) != 1:
+        logger.error(f'Could not determine assembly accession to check against fasta file, out of: {assemblies}')
+        return all_analyses, None
+    return all_analyses, assemblies[0]
 
 
 def main():
@@ -156,4 +150,3 @@ def main():
     analyses, metadata_insdc = get_analyses_and_reference_genome_from_metadata(args.vcf_files, args.metadata_json)
     results = assess_fasta(args.input_fasta, analyses, metadata_insdc)
     write_result_yaml(args.output_yaml, results)
-

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.8}/eva_sub_cli/executables/samples_checker.py

@@ -1,6 +1,4 @@
 import argparse
-import gzip
-import json
 import os
 
 from ebi_eva_common_pyutils.logger import logging_config
@@ -8,7 +6,7 @@ from ebi_eva_common_pyutils.logger import logging_config
 import yaml
 
 from eva_sub_cli.file_utils import open_gzip_if_required
-from eva_sub_cli.metadata_utils import get_samples_per_analysis, get_files_per_analysis, get_analysis_for_vcf_file
+from eva_sub_cli.metadata import EvaMetadataJson
 
 logger = logging_config.get_logger(__name__)
 
@@ -24,8 +22,10 @@ def get_samples_from_vcf(vcf_file):
                 if len(sp_line) > 9:
                     return sp_line[9:]
                 else:
-                    logger.warning(f"No Sample names found in file {vcf_file}")
+                    logger.warning(f"No sample names found in file {vcf_file}")
                     return []
+        logger.warning(f"No sample names found in file {vcf_file}")
+        return []
 
 
 def compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_file):
@@ -54,12 +54,12 @@ def compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_
             more_metadata_submitted_files)
 
 
-def compare_all_analysis(samples_per_analysis, files_per_analysis):
+def compare_all_analysis(metadata, files_per_analysis):
     overall_differences = False
     results_per_analysis_alias = {}
-    all_analysis_alias = set(samples_per_analysis) | set(files_per_analysis)
+    all_analysis_alias = set(metadata.samples_per_analysis) | set(files_per_analysis)
     for analysis_alias in all_analysis_alias:
-        sample_name_in_analysis = samples_per_analysis.get(analysis_alias, [])
+        sample_name_in_analysis = metadata.samples_per_analysis.get(analysis_alias, [])
         sample_name_per_file = {
             file_path: get_samples_from_vcf(file_path)
             for file_path in files_per_analysis.get(analysis_alias, [])
@@ -78,13 +78,7 @@ def compare_all_analysis(samples_per_analysis, files_per_analysis):
     return overall_differences, results_per_analysis_alias
 
 
-def read_metadata_json(json_file):
-    with open(json_file) as open_json:
-        metadata = json.load(open_json)
-        return get_samples_per_analysis(metadata), get_files_per_analysis(metadata)
-
-
-def associate_vcf_path_with_analysis(vcf_files, files_per_analysis):
+def associate_vcf_path_with_analysis(metadata, vcf_files):
     """
     Match the files names associated with analysis provided in the metadata with the file path given on the command
     line.
@@ -93,10 +87,10 @@ def associate_vcf_path_with_analysis(vcf_files, files_per_analysis):
     :returns dictionary of analysis and their associated vcf file path
     """
     result_files_per_analysis = dict()
-    for analysis in files_per_analysis:
+    for analysis in metadata.files_per_analysis:
         result_files_per_analysis[analysis] = []
     for vcf_file in vcf_files:
-        analysis_aliases = get_analysis_for_vcf_file(vcf_file, files_per_analysis)
+        analysis_aliases = metadata.get_analysis_for_vcf_file(vcf_file)
         if len(analysis_aliases) == 1:
             result_files_per_analysis[analysis_aliases[0]].append(vcf_file)
         elif len(analysis_aliases) == 0:
@@ -123,9 +117,9 @@ def check_sample_name_concordance(metadata_json, vcf_files, output_yaml):
     Take the metadata following EVA standard and formatted in JSON then compare the sample names in it to the ones
     found in the VCF files
     """
-    samples_per_analysis, files_per_analysis = read_metadata_json(metadata_json)
-    file_path_per_analysis = associate_vcf_path_with_analysis(vcf_files, files_per_analysis)
-    overall_differences, results_per_analysis_alias = compare_all_analysis(samples_per_analysis, file_path_per_analysis)
+    metadata = EvaMetadataJson(metadata_json)
+    file_path_per_analysis = associate_vcf_path_with_analysis(metadata, vcf_files)
+    overall_differences, results_per_analysis_alias = compare_all_analysis(metadata, file_path_per_analysis)
     write_result_yaml(output_yaml, overall_differences, results_per_analysis_alias)
 
 

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.8}/eva_sub_cli/executables/trim_down.py

@@ -1,6 +1,7 @@
 import argparse
 import os
 
+import sys
 import yaml
 from ebi_eva_common_pyutils.logger import logging_config
 from eva_sub_cli.file_utils import open_gzip_if_required, fasta_iter
@@ -8,15 +9,16 @@ from eva_sub_cli.file_utils import open_gzip_if_required, fasta_iter
 logger = logging_config.get_logger(__name__)
 
 
-max_nb_lines = 10000
+MAX_NB_LINES = 10000
 
 
-def trim_down_vcf(vcf_file, output_vcf):
+def trim_down_vcf(vcf_file, output_vcf, max_nb_lines=MAX_NB_LINES):
     """
     Produce a smaller vcf files containing a maximum of 10000 records
     """
-    with open_gzip_if_required(vcf_file) as vcf_in, open(output_vcf, 'w') as vcf_out:
+    with open_gzip_if_required(vcf_file) as vcf_in, open_gzip_if_required(output_vcf, 'w') as vcf_out:
         line_count = 0
+        trimmed_down = False
         ref_seq_names = set()
         for line in vcf_in:
             if line.startswith('#') or line_count < max_nb_lines:
@@ -25,10 +27,12 @@ def trim_down_vcf(vcf_file, output_vcf):
                     line_count += 1
                     ref_seq_names.add(line.split('\t')[0])
             else:
+                trimmed_down = True
                 break
     if line_count != max_nb_lines:
         logger.warning(f'Only {line_count} found in the source VCF {vcf_file} ')
-    return line_count, ref_seq_names
+
+    return line_count, trimmed_down, ref_seq_names
 
 
 def trim_down_fasta(fasta_file, output_fasta, ref_seq_names):
@@ -46,10 +50,9 @@ def trim_down_fasta(fasta_file, output_fasta, ref_seq_names):
                     print(sequence[i:i+80], file=fasta_out)
     return found_sequences
 
-
-def main():
+def parse_args(cmd_line_args):
     arg_parser = argparse.ArgumentParser(
-        description=f'Take a VCF file and only keep {max_nb_lines} lines and remove unused fasta sequence from the '
+        description=f'Take a VCF file and only keep {MAX_NB_LINES} lines and remove unused fasta sequence from the '
                     f'associated reference genome')
     arg_parser.add_argument('--vcf_file', dest='vcf_file', required=True,
                             help='Path to the vcf file to be trimmed down')
@@ -63,16 +66,23 @@ def main():
                             help='Path to the yaml file containing the trim down metrics')
 
     args = arg_parser.parse_args()
+    return args
+
+def main():
+    args = parse_args(sys.argv[1:])
+
     logging_config.add_stdout_handler()
 
-    line_count, ref_sequence = trim_down_vcf(args.vcf_file, args.output_vcf_file)
+    line_count, trimmed_down, ref_sequence = trim_down_vcf(args.vcf_file, args.output_vcf_file)
     sequence_found = trim_down_fasta(args.fasta_file, args.output_fasta_file, ref_sequence)
-    trim_down_metrics = {'trim_down_vcf_record': line_count, 'number_sequence_found': sequence_found,
-                         'trim_down_required': line_count == max_nb_lines}
+    trim_down_metrics = {'trim_down_vcf_record': line_count, 'number_sequence_found': len(sequence_found),
+                         'trim_down_required': trimmed_down}
     if len(sequence_found) != len(ref_sequence):
         logger.warning(f'Not all sequences were found in the fasta file. Cancelling trimming down of fasta file')
+        if os.path.exists(args.output_fasta_file):
+            os.remove(args.output_fasta_file)
         os.link(args.fasta_file, args.output_fasta_file)
         trim_down_metrics.pop('number_sequence_found')
-    with open(args.output_yaml_file) as open_file:
+    with open(args.output_yaml_file, 'w') as open_file:
         yaml.safe_dump(trim_down_metrics, open_file)
 

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.8}/eva_sub_cli/executables/xlsx2json.py

@@ -105,11 +105,11 @@ class XlsxParser:
             if type_name == 'string':
                 return str(value)
             if type_name == 'boolean':
-                return str(value).lower() in ['true', '1', 't', 'y', 'yes']
+                return str(value).lower() in ['true', '1', "'1'", 't', 'y', 'yes']
             if type_name == 'list':
                 # split and remove empty values
-                return [XlsxParser.trim_value(v)
-                        for v in value.split(',') if v]
+                split_values = [XlsxParser.trim_value(v) for v in value.split(',') if v]
+                return [v for v in split_values if v]
             if type_name == 'date':
                 return XlsxParser.serialize(value)
         return value

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.8}/eva_sub_cli/file_utils.py

@@ -23,6 +23,11 @@ def is_submission_dir_writable(submission_dir):
         return False
     return True
 
+def is_vcf_file(file_path):
+    if file_path:
+        file_path = file_path.strip().lower()
+        return file_path.endswith('.vcf') or file_path.endswith('.vcf.gz')
+    return False
 
 def backup_file_or_directory(file_name, max_backups=None):
     """
@@ -46,12 +51,12 @@ def backup_file_or_directory(file_name, max_backups=None):
     os.rename(file_name, file_name + '.1')
 
 
-def open_gzip_if_required(input_file):
+def open_gzip_if_required(input_file, mode='r'):
     """Open a file in read mode using gzip if the file extension says .gz"""
     if input_file.endswith('.gz'):
-        return gzip.open(input_file, 'rt')
+        return gzip.open(input_file, mode + 't')
     else:
-        return open(input_file, 'r')
+        return open(input_file, mode)
 
 
 def fasta_iter(input_fasta):

eva_sub_cli-0.4.8/eva_sub_cli/metadata.py

@@ -0,0 +1,95 @@
+import json
+import os
+from collections import defaultdict
+from copy import deepcopy
+from functools import cached_property
+
+from ebi_eva_common_pyutils.logger import AppLogger
+
+
+class EvaMetadataJson(AppLogger):
+
+    def __init__(self, path_to_json):
+        with open(path_to_json) as open_json:
+            self.content = json.load(open_json)
+
+    @property
+    def project(self):
+        return self.content.get('project', {})
+
+    @property
+    def submitter_details(self):
+        return self.content.get('submitterDetails', {})
+
+    @property
+    def analyses(self):
+        return self.content.get('analysis', [])
+
+    @property
+    def samples(self):
+        return self.content.get('sample', [])
+
+    @property
+    def files(self):
+        return self.content.get('files', [])
+
+    @cached_property
+    def resolved_files(self):
+        """Returns list of files with fileName resolved to an absolute path."""
+        mod_files = deepcopy(self.content.get('files', []))
+        for file_info in mod_files:
+            if 'fileName' in file_info:
+                file_info['fileName'] = os.path.abspath(file_info['fileName'])
+        return mod_files
+
+    def set_files(self, file_list):
+        assert isinstance(file_list, list)
+        self.content['files'] = file_list
+        # invalidate the cached property
+        if self.resolved_files:
+            del self.resolved_files
+
+    @cached_property
+    def samples_per_analysis(self):
+        """Returns mapping of analysis alias to sample names, based on metadata."""
+        samples_per_analysis = defaultdict(list)
+        for sample_info in self.samples:
+            for analysis_alias in sample_info.get('analysisAlias', []):
+                samples_per_analysis[analysis_alias].append(sample_info.get('sampleInVCF'))
+        return {
+            analysis_alias: set(samples)
+            for analysis_alias, samples in samples_per_analysis.items()
+        }
+
+    @cached_property
+    def files_per_analysis(self):
+        """Returns mapping of analysis alias to filenames, based on metadata."""
+        files_per_analysis = defaultdict(list)
+        for file_info in self.resolved_files:
+            files_per_analysis[file_info.get('analysisAlias')].append(file_info.get('fileName'))
+        return {
+            analysis_alias: set(filepaths)
+            for analysis_alias, filepaths in files_per_analysis.items()
+        }
+
+    def get_reference_assembly_for_analysis(self, analysis_alias):
+        """Returns the reference assembly for this analysis (does not validate format)."""
+        for analysis in self.analyses:
+            if analysis.get('analysisAlias') == analysis_alias:
+                return analysis.get('referenceGenome')
+        return None
+
+    def get_analysis_for_vcf_file(self, vcf_file):
+        """Returns list of analysis aliases associated with the vcf file path."""
+        if not os.path.exists(vcf_file):
+            raise FileNotFoundError(f'{vcf_file} cannot be resolved')
+        analysis_aliases = [analysis_alias for analysis_alias in self.files_per_analysis
+                            if vcf_file in self.files_per_analysis[analysis_alias]
+                            or os.path.basename(vcf_file) in [
+                                os.path.basename(p) for p in self.files_per_analysis[analysis_alias]
+                            ]]
+        return analysis_aliases
+
+    def write(self, output_path):
+        with open(output_path, 'w') as open_file:
+            json.dump(self.content, open_file)

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.8}/eva_sub_cli/nextflow/validation.nf

@@ -116,6 +116,7 @@ process trim_down_vcf {
     path "${vcf.getBaseName()}_trim_down.log", emit: trim_down_log
     path "${vcf.getBaseName()}_trim_down.yml", emit: trim_down_metric
 
+    script:
     """
     mkdir output
     $params.python_scripts.trim_down --vcf_file $vcf  --output_vcf_file output/$vcf --fasta_file $fasta --output_fasta_file output/$fasta --output_yaml_file ${vcf.getBaseName()}_trim_down.yml > ${vcf.getBaseName()}_trim_down.log