eva-sub-cli 0.4.7__tar.gz → 0.4.9__tar.gz

eva_sub_cli-0.4.9/.github/workflows/trigger-eva-sub-cli-integration-tests.yml

@@ -0,0 +1,16 @@
+name: Trigger Eva Sub Cli Integration Tests
+
+on:
+  push:
+    branches: [main, test-trigger-integration-tests]
+
+jobs:
+  trigger-integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger eva sub cli integration tests
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          token: ${{ secrets.TRIGGER_INTEGRATION_TESTS }}
+          repository: EBIvariation/eva-integration-tests
+          event-type: trigger-eva-sub-cli-tests

{eva_sub_cli-0.4.7/eva_sub_cli.egg-info → eva_sub_cli-0.4.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.4.7
+Version: 0.4.9
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -19,7 +19,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.1
+Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.3
 Requires-Dist: jinja2==3.*,>=3.0.0
 Requires-Dist: jsonschema==4.*
 Requires-Dist: openpyxl==3.*

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.9}/eva_sub_cli/__init__.py

@@ -6,7 +6,7 @@ PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__))
 NEXTFLOW_DIR = os.path.join(PACKAGE_DIR, 'nextflow')
 ETC_DIR = os.path.join(PACKAGE_DIR, 'etc')
 LSRI_CLIENT_ID = "aa0fcc42-096a-4f9d-b871-aceb1a97d174"
-DEFAULT_METADATA_XLSX_TEMPLATE_VERSION = '1.1.6'
+MINIMUM_METADATA_XLSX_TEMPLATE_VERSION = '2.0.0'
 
 SUB_CLI_CONFIG_FILE = ".eva_sub_cli_config.yml"
 

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.9}/eva_sub_cli/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.7'
-__version_tuple__ = version_tuple = (0, 4, 7)
+__version__ = version = '0.4.9'
+__version_tuple__ = version_tuple = (0, 4, 9)

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.9}/eva_sub_cli/etc/eva_schema.json

@@ -310,9 +310,10 @@
             "type": "string",
             "description": "Sample Name used in the VCF file"
           },
-          "sampleAccession": {
+          "bioSampleAccession": {
             "type": "string",
-            "description": "Accession of the sample (BioSamples, ENA or EGA)"
+            "pattern": "^SAM(E|D|N)[A-Z]?[0-9]+$",
+            "description": "BioSample accession of pre-existing sample. Only use for preregistered samples"
           },
           "bioSampleObject": {
             "type": "object",

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.9}/eva_sub_cli/executables/check_fasta_insdc.py

@@ -1,8 +1,5 @@
 import argparse
-import gzip
 import hashlib
-import json
-from itertools import groupby
 
 import requests
 import yaml
@@ -11,8 +8,7 @@ from requests import HTTPError
 from retry import retry
 
 from eva_sub_cli.file_utils import fasta_iter
-from eva_sub_cli.metadata_utils import get_files_per_analysis, get_analysis_for_vcf_file, \
-    get_reference_assembly_for_analysis
+from eva_sub_cli.metadata import EvaMetadataJson
 
 REFGET_SERVER = 'https://www.ebi.ac.uk/ena/cram'
 CONTIG_ALIAS_SERVER = 'https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum'
@@ -52,7 +48,7 @@ def _get_containing_assemblies_paged(url):
                 results.add(contigEntity['assembly']['insdcAccession'])
         if '_links' in response_data and 'next' in response_data['_links']:
             # Add results from next page if needed
-            results |= _get_containing_assemblies_paged(response_data['_links']['next'])
+            results = results.union(_get_containing_assemblies_paged(response_data['_links']['next']['href']))
         return results
     return set()
 
@@ -94,7 +90,7 @@ def assess_fasta(input_fasta, analyses, assembly_in_metadata):
                 possible_assemblies = containing_assemblies
             else:
                 possible_assemblies &= containing_assemblies
-    except (ConnectionError, HTTPError) as e:
+    except Exception as e:
         # Server errors from either ENA refget or EVA contig alias will halt the check prematurely.
         # Report the error but do not return from the method, so that incomplete results can be reported
         # (i.e. any sequences found to be INSDC and any compatible assemblies so far)
@@ -121,23 +117,21 @@ def get_analyses_and_reference_genome_from_metadata(vcf_files_for_fasta, json_fi
       single assembly accession
     :param json_file: JSON file of the metadata
     """
-    with open(json_file) as open_json:
-        metadata = json.load(open_json)
-        files_per_analysis = get_files_per_analysis(metadata)
-        # Get all analyses associated with all vcf files that are linked with a single fasta file
-        all_analyses = set()
-        for vcf_file in vcf_files_for_fasta:
-            analysis_aliases = get_analysis_for_vcf_file(vcf_file, files_per_analysis)
-            if len(analysis_aliases) != 1:
-                logger.error(f'Could not determine analysis associated with VCF file: {vcf_file}')
-            else:
-                all_analyses.add(analysis_aliases[0])
-        # Get (single) assembly associated with these analyses
-        assemblies = [get_reference_assembly_for_analysis(metadata, analysis) for analysis in all_analyses]
-        if len(assemblies) != 1:
-            logger.error(f'Could not determine assembly accession to check against fasta file, out of: {assemblies}')
-            return all_analyses, None
-        return all_analyses, assemblies[0]
+    metadata = EvaMetadataJson(json_file)
+    # Get all analyses associated with all vcf files that are linked with a single fasta file
+    all_analyses = set()
+    for vcf_file in vcf_files_for_fasta:
+        analysis_aliases = metadata.get_analysis_for_vcf_file(vcf_file)
+        if len(analysis_aliases) != 1:
+            logger.error(f'Could not determine analysis associated with VCF file: {vcf_file}')
+        else:
+            all_analyses.add(analysis_aliases[0])
+    # Get (single) assembly associated with these analyses
+    assemblies = [metadata.get_reference_assembly_for_analysis(analysis) for analysis in all_analyses]
+    if len(assemblies) != 1:
+        logger.error(f'Could not determine assembly accession to check against fasta file, out of: {assemblies}')
+        return all_analyses, None
+    return all_analyses, assemblies[0]
 
 
 def main():
@@ -156,4 +150,3 @@ def main():
     analyses, metadata_insdc = get_analyses_and_reference_genome_from_metadata(args.vcf_files, args.metadata_json)
     results = assess_fasta(args.input_fasta, analyses, metadata_insdc)
     write_result_yaml(args.output_yaml, results)
-

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.9}/eva_sub_cli/executables/samples_checker.py

@@ -1,6 +1,4 @@
 import argparse
-import gzip
-import json
 import os
 
 from ebi_eva_common_pyutils.logger import logging_config
@@ -8,7 +6,7 @@ from ebi_eva_common_pyutils.logger import logging_config
 import yaml
 
 from eva_sub_cli.file_utils import open_gzip_if_required
-from eva_sub_cli.metadata_utils import get_samples_per_analysis, get_files_per_analysis, get_analysis_for_vcf_file
+from eva_sub_cli.metadata import EvaMetadataJson
 
 logger = logging_config.get_logger(__name__)
 
@@ -24,8 +22,10 @@ def get_samples_from_vcf(vcf_file):
                 if len(sp_line) > 9:
                     return sp_line[9:]
                 else:
-                    logger.warning(f"No Sample names found in file {vcf_file}")
+                    logger.warning(f"No sample names found in file {vcf_file}")
                     return []
+        logger.warning(f"No sample names found in file {vcf_file}")
+        return []
 
 
 def compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_file):
@@ -54,12 +54,12 @@ def compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_
             more_metadata_submitted_files)
 
 
-def compare_all_analysis(samples_per_analysis, files_per_analysis):
+def compare_all_analysis(metadata, files_per_analysis):
     overall_differences = False
     results_per_analysis_alias = {}
-    all_analysis_alias = set(samples_per_analysis) | set(files_per_analysis)
+    all_analysis_alias = set(metadata.samples_per_analysis) | set(files_per_analysis)
     for analysis_alias in all_analysis_alias:
-        sample_name_in_analysis = samples_per_analysis.get(analysis_alias, [])
+        sample_name_in_analysis = metadata.samples_per_analysis.get(analysis_alias, [])
         sample_name_per_file = {
             file_path: get_samples_from_vcf(file_path)
             for file_path in files_per_analysis.get(analysis_alias, [])
@@ -78,13 +78,7 @@ def compare_all_analysis(samples_per_analysis, files_per_analysis):
     return overall_differences, results_per_analysis_alias
 
 
-def read_metadata_json(json_file):
-    with open(json_file) as open_json:
-        metadata = json.load(open_json)
-        return get_samples_per_analysis(metadata), get_files_per_analysis(metadata)
-
-
-def associate_vcf_path_with_analysis(vcf_files, files_per_analysis):
+def associate_vcf_path_with_analysis(metadata, vcf_files):
     """
     Match the files names associated with analysis provided in the metadata with the file path given on the command
     line.
@@ -93,10 +87,10 @@ def associate_vcf_path_with_analysis(vcf_files, files_per_analysis):
     :returns dictionary of analysis and their associated vcf file path
     """
     result_files_per_analysis = dict()
-    for analysis in files_per_analysis:
+    for analysis in metadata.files_per_analysis:
         result_files_per_analysis[analysis] = []
     for vcf_file in vcf_files:
-        analysis_aliases = get_analysis_for_vcf_file(vcf_file, files_per_analysis)
+        analysis_aliases = metadata.get_analysis_for_vcf_file(vcf_file)
         if len(analysis_aliases) == 1:
             result_files_per_analysis[analysis_aliases[0]].append(vcf_file)
         elif len(analysis_aliases) == 0:
@@ -123,9 +117,9 @@ def check_sample_name_concordance(metadata_json, vcf_files, output_yaml):
     Take the metadata following EVA standard and formatted in JSON then compare the sample names in it to the ones
     found in the VCF files
     """
-    samples_per_analysis, files_per_analysis = read_metadata_json(metadata_json)
-    file_path_per_analysis = associate_vcf_path_with_analysis(vcf_files, files_per_analysis)
-    overall_differences, results_per_analysis_alias = compare_all_analysis(samples_per_analysis, file_path_per_analysis)
+    metadata = EvaMetadataJson(metadata_json)
+    file_path_per_analysis = associate_vcf_path_with_analysis(metadata, vcf_files)
+    overall_differences, results_per_analysis_alias = compare_all_analysis(metadata, file_path_per_analysis)
     write_result_yaml(output_yaml, overall_differences, results_per_analysis_alias)
 
 

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.9}/eva_sub_cli/executables/xlsx2json.py

@@ -105,11 +105,11 @@ class XlsxParser:
             if type_name == 'string':
                 return str(value)
             if type_name == 'boolean':
-                return str(value).lower() in ['true', '1', 't', 'y', 'yes']
+                return str(value).lower() in ['true', '1', "'1'", 't', 'y', 'yes']
             if type_name == 'list':
                 # split and remove empty values
-                return [XlsxParser.trim_value(v)
-                        for v in value.split(',') if v]
+                split_values = [XlsxParser.trim_value(v) for v in value.split(',') if v]
+                return [v for v in split_values if v]
             if type_name == 'date':
                 return XlsxParser.serialize(value)
         return value

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.9}/eva_sub_cli/file_utils.py

@@ -23,6 +23,11 @@ def is_submission_dir_writable(submission_dir):
         return False
     return True
 
+def is_vcf_file(file_path):
+    if file_path:
+        file_path = file_path.strip().lower()
+        return file_path.endswith('.vcf') or file_path.endswith('.vcf.gz')
+    return False
 
 def backup_file_or_directory(file_name, max_backups=None):
     """

eva_sub_cli-0.4.7/eva_sub_cli/jinja_templates/html_report.html → eva_sub_cli-0.4.9/eva_sub_cli/jinja_templates/html/report.html

@@ -69,11 +69,11 @@
 <section>
     <h2>VCF validation results</h2>
     <div class="description">
-        Checks whether each file is compliant with the <a href="http://samtools.github.io/hts-specs/VCFv4.3.pdf" target=”_blank”>VCF specification</a>.
+        Checks whether each file is compliant with the <a href="http://samtools.github.io/hts-specs/VCFv4.4.pdf" target=”_blank”>VCF specification</a>.
         Also checks whether the variants' reference alleles match against the reference assembly.
     </div>
     {% for file_name in vcf_files %}
-        {% if file_name!= "pass"%}
+        {% if file_name != "pass"%}
         <h3>{{ file_name }}</h3>
         {{ file_validation_report(validation_results, file_name) }}
         {% endif %}
@@ -95,7 +95,7 @@
         Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file.
     </div>
     {% for file_name in fasta_files %}
-        {% if file_name!= "pass"%}
+        {% if file_name != "pass"%}
         <h3>{{ file_name }}</h3>
         {{ fasta_check_report(validation_results, file_name) }}
         {% endif %}

eva_sub_cli-0.4.9/eva_sub_cli/jinja_templates/text/fasta_check.txt

@@ -0,0 +1,46 @@
+{% macro fasta_check_report(validation_results, file_name) -%}
+
+	{% set results_for_fasta = validation_results['fasta_check'][file_name] %}
+
+	{% if results_for_fasta.get('connection_error') %}
+		Warning: The following results may be incomplete due to problems with external services.
+		Please try again later for complete results.
+		Error message: {{ results_for_fasta.get('connection_error') }}
+	{% endif %}
+
+	{% if results_for_fasta.get('all_insdc') %}
+		{{ "\u2714" }} All sequences are INSDC accessioned.
+	{% else %}
+		{{ "\u274C" }} Some sequences are not INSDC accessioned
+			First 10 sequences not in INSDC. Full report: {{ results_for_fasta.get('report_path', '') }}
+			{% set sequence_info_list = results_for_fasta.get('sequences', [])|rejectattr("insdc")|list %}
+			{% for sequence_info in sequence_info_list[:10] %}
+				Sequence name: {{ sequence_info.get('sequence_name') }}
+				Refget md5: {{ sequence_info.get('sequence_md5') }}
+				---
+			{% endfor %}
+	{% endif %}
+
+	{% if 'possible_assemblies' in results_for_fasta %}
+		{% if 'metadata_assembly_compatible' in results_for_fasta %}
+			{% set analysis_text = results_for_fasta.get('associated_analyses')|join(", ") %}
+			{% if results_for_fasta.get('metadata_assembly_compatible') %}
+				{% set icon = "\u2714" %}
+				{% set text = analysis_text + ": Assembly accession in metadata is compatible" %}
+			{% else %}
+				{% set icon = "\u274C" %}
+				{% set text = analysis_text + ": Assembly accession in metadata is not compatible" %}
+			{% endif %}
+		{% else %}
+			{% set icon = "\u274C" %}
+			{% set text = "No assembly accession found in metadata" %}
+		{% endif %}
+		{{ icon }} {{ text }}
+		{% if 'metadata_assembly_compatible' not in results_for_fasta or not results_for_fasta['metadata_assembly_compatible'] %}
+			Full report: {{ results_for_fasta.get('report_path', '') }}
+			Assembly accession found in metadata: {{ results_for_fasta.get('assembly_in_metadata', 'Not found') }}
+			Assembly accession(s) compatible with FASTA: {{ results_for_fasta.get('possible_assemblies')|join(", ") }}
+		{% endif %}
+	{% endif %}
+
+{%- endmacro %}

eva_sub_cli-0.4.9/eva_sub_cli/jinja_templates/text/file_validation.txt

@@ -0,0 +1,58 @@
+
+{% macro file_validation_report(validation_results, file_name) -%}
+    {% for check_type, check_per_file in validation_results.items() %}
+        {% set result = check_per_file.get(file_name, {}) %}
+        {% if check_type == "assembly_check" %}
+            {{ assembly_check(result) }}
+        {% elif check_type == "vcf_check" %}
+            {{ vcf_check(result) }}
+        {% endif %}
+    {% endfor %}
+{%- endmacro %}
+
+{% macro vcf_check(vcf_check_result) %}
+    {% set critical_count = vcf_check_result.get("critical_count", 0) %}
+    {% set error_count = vcf_check_result.get("error_count", 0) %}
+    {% if critical_count > 0 or error_count > 0 %}
+        {% set icon = "\u274C" %}
+    {% else %}
+        {% set icon = "\u2714" %}
+    {% endif %}
+		{{ icon }} VCF check: {{ critical_count }} critical errors, {{ error_count }} non-critical errors
+
+	{% set critical_list = vcf_check_result.get("critical_list") %}
+	{% set error_list = vcf_check_result.get("error_list") %}
+	{% if critical_list or error_list%}
+		First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }}
+		{% for error in critical_list[:10] %}
+			Critical error: {{ error }}
+		{% endfor %}
+		{% for error in error_list[:10] %}
+			Non-critical error: {{ error }}
+		{% endfor %}
+	{% endif %}
+{%- endmacro %}
+
+{% macro assembly_check(assembly_check_result) %}
+    {% set nb_match = assembly_check_result.get("match", 0) %}
+    {% set nb_total = assembly_check_result.get("total", 0) %}
+    {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %}
+    {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %}
+        {% set icon = "\u274C" %}
+    {% else %}
+        {% set icon = "\u2714" %}
+    {% endif %}
+		{{ icon }} Assembly check: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%)
+
+	{% set mismatch_list = assembly_check_result.get("mismatch_list") %}
+	{% set error_list = assembly_check_result.get("error_list") %}
+	{% if mismatch_list or error_list %}
+		First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }}
+		{% for error in error_list[:10] %}
+			Parsing error: {{ error }}
+		{% endfor %}
+		{% for error in mismatch_list[:10] %}
+			Mismatch error: {{ error }}
+		{% endfor %}
+	{% endif %}
+{%- endmacro %}

eva_sub_cli-0.4.9/eva_sub_cli/jinja_templates/text/metadata_validation.txt

@@ -0,0 +1,33 @@
+
+{% macro metadata_validation_report(validation_results) -%}
+
+    {% set results = validation_results.get('metadata_check', {}) %}
+    {% set spreadsheet_errors = results.get('spreadsheet_errors', []) %}
+    {% set json_errors = results.get('json_errors', []) %}
+    {% set has_errors = spreadsheet_errors or json_errors %}
+    {% if has_errors %}
+        {% set icon = "\u274C" %}
+    {% else %}
+        {% set icon = "\u2714" %}
+    {% endif %}
+    {{ icon }} Metadata validation check
+
+{% if spreadsheet_errors %}
+	Full report: {{ results.get('spreadsheet_report_path', '') }}
+	{% for error in spreadsheet_errors %}
+		Sheet: {{ error.get('sheet') }} | Row: {{ error.get('row') }} | Column: {{ error.get('column') }}
+		Error: {{ error.get('description') }}
+		---
+	{% endfor %}
+{% endif %}
+
+{% if json_errors %}
+	Full report: {{ results.get('json_report_path', '') }}
+	{% for error in json_errors %}
+		JSON Property: {{ error.get('property') }}
+		Error: {{ error.get('description') }}
+		---
+	{% endfor %}
+{% endif %}
+
+{%- endmacro %}

eva_sub_cli-0.4.9/eva_sub_cli/jinja_templates/text/project_details.txt

@@ -0,0 +1,21 @@
+{% macro project_details_report(project_title, validation_date, submission_dir, vcf_fasta_analysis_mapping) -%}
+
+{% if project_title %}
+	Project Title: {{ project_title }}
+{% else %}
+	Project Title: Not Found
+{% endif %}
+	Validation Date: {{ validation_date }}
+	Submission Directory: {{ submission_dir }}
+
+{% if vcf_fasta_analysis_mapping %}
+	Files mapping:
+	{% for mapping in vcf_fasta_analysis_mapping %}
+		---
+		VCF File: {{ mapping.vcf_file }}
+		Fasta File: {{ mapping.fasta_file }}
+		Analysis: {{ mapping.analysis }}
+	{% endfor %}
+{% endif %}
+
+{%- endmacro %}

eva_sub_cli-0.4.9/eva_sub_cli/jinja_templates/text/report.txt

@@ -0,0 +1,59 @@
+{% from 'project_details.txt' import project_details_report %}
+{% from 'file_validation.txt' import file_validation_report %}
+{% from 'sample_name_check.txt' import sample_name_check_report %}
+{% from 'fasta_check.txt' import fasta_check_report %}
+{% from 'metadata_validation.txt' import metadata_validation_report %}
+{% from 'shallow_validation.txt' import optional_shallow_validation_report %}
+
+VALIDATION REPORT
+eva-sub-cli v{{cli_version}}
+
+{{ optional_shallow_validation_report(validation_results) }}
+
+-
+
+PROJECT SUMMARY
+General details about the project
+
+{{ project_details_report(project_title, validation_date, submission_dir, vcf_fasta_analysis_mapping) }}
+
+-
+
+METADATA VALIDATION RESULTS
+Ensures that required fields are present and values are formatted correctly.
+For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data).
+
+{{ metadata_validation_report(validation_results) }}
+
+-
+
+VCF VALIDATION RESULTS
+Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf).
+Also checks whether the variants' reference alleles match against the reference assembly.
+
+{% for file_name in vcf_files %}
+{% if file_name != "pass"%}
+	{{ file_name }}
+	{{ file_validation_report(validation_results, file_name) }}
+{% endif %}
+{% endfor %}
+
+-
+
+SAMPLE NAME CONCORDANCE CHECK
+Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names.
+
+{{ sample_name_check_report(validation_results)}}
+
+-
+
+REFERENCE GENOME INSDC CHECK
+Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC.
+Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file.
+
+{% for file_name in fasta_files %}
+{% if file_name != "pass"%}
+	{{ file_name }}
+	{{ fasta_check_report(validation_results, file_name) }}
+{% endif %}
+{% endfor %}

eva_sub_cli-0.4.9/eva_sub_cli/jinja_templates/text/sample_name_check.txt

@@ -0,0 +1,24 @@
+
+{% macro sample_name_check_report(validation_results) -%}
+    {% set results = validation_results.get('sample_check', {}) %}
+    {% macro format_error(error) %}
+        {{ error | replace(' ', '•') }}
+    {% endmacro %}
+
+{% for analysis, results_for_analysis in results.get('results_per_analysis', {}).items() %}
+	{% if results_for_analysis.get('difference') %}
+		{% set icon = "\u274C" %}
+		{% set text = "Sample names in metadata do not match with those in VCF files" %}
+	{% else %}
+		{% set icon = "\u2714" %}
+		{% set text = "Sample names in metadata match with those in VCF files" %}
+	{% endif %}
+	{{ icon }} {{ analysis }}: {{ text }}
+
+	{% if results_for_analysis.get('difference') %}
+		First 10 errors per category are below. Full report: {{ results.get('report_path', '') }}
+			Samples described in the metadata but not in the VCF files: {{ results_for_analysis.get('more_metadata_submitted_files')[:10]|join(", ") }}
+			Samples in the VCF files but not described in the metadata: {{ results_for_analysis.get('more_submitted_files_metadata')[:10]|join(", ") }}
+	{% endif %}
+{% endfor %}
+{%- endmacro %}

eva_sub_cli-0.4.9/eva_sub_cli/jinja_templates/text/shallow_validation.txt

@@ -0,0 +1,16 @@
+{% macro optional_shallow_validation_report(validation_results) -%}
+
+{% set results = validation_results.get('shallow_validation', {}) %}
+{% if results.get('required') %}
+
+{{ "\u274C" }} You requested to run the shallow validation, please run full validation before submitting the data
+{% for vcf_file in results.get('metrics') %}
+	---
+	VCF File: {{ vcf_file }}
+	Variant lines validated in VCF: {{ results.get('metrics').get(vcf_file).get('trim_down_vcf_record') }}
+	Entries used in Fasta: {{ results.get('metrics').get(vcf_file).get('number_sequence_found') }}
+{% endfor %}
+
+{% endif %}
+
+{%- endmacro %}

eva_sub_cli-0.4.9/eva_sub_cli/metadata.py

@@ -0,0 +1,95 @@
+import json
+import os
+from collections import defaultdict
+from copy import deepcopy
+from functools import cached_property
+
+from ebi_eva_common_pyutils.logger import AppLogger
+
+
+class EvaMetadataJson(AppLogger):
+
+    def __init__(self, path_to_json):
+        with open(path_to_json) as open_json:
+            self.content = json.load(open_json)
+
+    @property
+    def project(self):
+        return self.content.get('project', {})
+
+    @property
+    def submitter_details(self):
+        return self.content.get('submitterDetails', {})
+
+    @property
+    def analyses(self):
+        return self.content.get('analysis', [])
+
+    @property
+    def samples(self):
+        return self.content.get('sample', [])
+
+    @property
+    def files(self):
+        return self.content.get('files', [])
+
+    @cached_property
+    def resolved_files(self):
+        """Returns list of files with fileName resolved to an absolute path."""
+        mod_files = deepcopy(self.content.get('files', []))
+        for file_info in mod_files:
+            if 'fileName' in file_info:
+                file_info['fileName'] = os.path.abspath(file_info['fileName'])
+        return mod_files
+
+    def set_files(self, file_list):
+        assert isinstance(file_list, list)
+        self.content['files'] = file_list
+        # invalidate the cached property
+        if self.resolved_files:
+            del self.resolved_files
+
+    @cached_property
+    def samples_per_analysis(self):
+        """Returns mapping of analysis alias to sample names, based on metadata."""
+        samples_per_analysis = defaultdict(list)
+        for sample_info in self.samples:
+            for analysis_alias in sample_info.get('analysisAlias', []):
+                samples_per_analysis[analysis_alias].append(sample_info.get('sampleInVCF'))
+        return {
+            analysis_alias: set(samples)
+            for analysis_alias, samples in samples_per_analysis.items()
+        }
+
+    @cached_property
+    def files_per_analysis(self):
+        """Returns mapping of analysis alias to filenames, based on metadata."""
+        files_per_analysis = defaultdict(list)
+        for file_info in self.resolved_files:
+            files_per_analysis[file_info.get('analysisAlias')].append(file_info.get('fileName'))
+        return {
+            analysis_alias: set(filepaths)
+            for analysis_alias, filepaths in files_per_analysis.items()
+        }
+
+    def get_reference_assembly_for_analysis(self, analysis_alias):
+        """Returns the reference assembly for this analysis (does not validate format)."""
+        for analysis in self.analyses:
+            if analysis.get('analysisAlias') == analysis_alias:
+                return analysis.get('referenceGenome')
+        return None
+
+    def get_analysis_for_vcf_file(self, vcf_file):
+        """Returns list of analysis aliases associated with the vcf file path."""
+        if not os.path.exists(vcf_file):
+            raise FileNotFoundError(f'{vcf_file} cannot be resolved')
+        analysis_aliases = [analysis_alias for analysis_alias in self.files_per_analysis
+                            if vcf_file in self.files_per_analysis[analysis_alias]
+                            or os.path.basename(vcf_file) in [
+                                os.path.basename(p) for p in self.files_per_analysis[analysis_alias]
+                            ]]
+        return analysis_aliases
+
+    def write(self, output_path):
+        with open(output_path, 'w') as open_file:
+            json.dump(self.content, open_file)