eva-sub-cli 0.6.4__tar.gz → 0.6.6__tar.gz

{eva_sub_cli-0.6.4 → eva_sub_cli-0.6.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.6.4
+Version: 0.6.6
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console

{eva_sub_cli-0.6.4 → eva_sub_cli-0.6.6}/eva_sub_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.6.4'
-__version_tuple__ = version_tuple = (0, 6, 4)
+__version__ = version = '0.6.6'
+__version_tuple__ = version_tuple = (0, 6, 6)
 
-__commit_id__ = commit_id = 'g1a3c8a4d1'
+__commit_id__ = commit_id = 'gdb6448e77'

{eva_sub_cli-0.6.4 → eva_sub_cli-0.6.6}/eva_sub_cli/call_home.py

@@ -1,4 +1,5 @@
 import os
+import threading
 import traceback
 import uuid
 from datetime import datetime, timezone
@@ -67,6 +68,7 @@ class CallHomeClient:
         self.run_id = _get_or_create_run_id(submission_dir)
         self.executor = executor
         self.tasks = tasks
+        self._threads = []
 
     def _build_payload(self, event_type, **kwargs):
         now = datetime.now(timezone.utc)
@@ -90,12 +92,22 @@ class CallHomeClient:
         return payload
 
     def _send_event(self, event_type, **kwargs):
+        payload = self._build_payload(event_type, **kwargs)
+        thread = threading.Thread(target=self._post_event, args=(event_type, payload))
+        thread.start()
+        self._threads.append(thread)
+
+    def _post_event(self, event_type, payload):
         try:
-            payload = self._build_payload(event_type, **kwargs)
-            requests.post(_get_call_home_url(), json=payload, timeout=5)
+            requests.post(_get_call_home_url(), json=payload, timeout=60)
         except Exception:
             logger.debug('Failed to send %s call-home event', event_type)
 
+    def wait_for_pending_events(self, timeout=None):
+        for thread in self._threads:
+            thread.join(timeout)
+        self._threads = []
+
     def send_start(self):
         self._send_event(EVENT_START)
 

{eva_sub_cli-0.6.4 → eva_sub_cli-0.6.6}/eva_sub_cli/etc/eva_schema.json

@@ -89,12 +89,14 @@
             "title": {
               "type": "string",
               "description": "Title of the project",
-              "maxLength": 500
+              "minLength": 20,
+              "maxLength": 250
             },
             "description": {
               "type": "string",
               "description": "Description of the project",
-              "maxLength": 5000
+              "minLength": 20,
+              "maxLength": 4000
             },
             "centre": {
               "type": "string",

eva_sub_cli-0.6.6/eva_sub_cli/jinja_templates/html/file_validation.html

@@ -0,0 +1,89 @@
+
+
+{% macro vcf_check_report(vcf_check_result_per_file, vcf_file_names) -%}
+    {% for file_name in vcf_file_names %}
+        {% set vcf_check_result = vcf_check_result_per_file.get(file_name, {}) %}
+        {% set critical_count = vcf_check_result.get("critical_count", 0) %}
+        {% set error_count = vcf_check_result.get("error_count", 0) %}
+        {% set expand_icon = "" %}
+        {% if critical_count > 0 %}
+            {% set expand_icon = "▶" %}
+            {% set icon = "❌" %}
+            {% set row_class = "report-section fail collapsible" %}
+        {% elif error_count > 0 %}
+            {% set expand_icon = "▶" %}
+            {% set icon = "❗" %}
+            {% set row_class = "report-section warn collapsible" %}
+        {% else %}
+            {% set icon = "✔" %}
+            {% set row_class = "report-section pass" %}
+        {% endif %}
+        <div class='{{ row_class }}'><span class="expand_icon">{{ expand_icon }}</span> {{ icon }} {{ file_name }}: {{ critical_count }} critical errors, {{ error_count }} non-critical errors </div>
+        {% set critical_list = vcf_check_result.get("critical_list") %}
+        {% set error_list = vcf_check_result.get("error_list") %}
+
+        {% if critical_list or error_list%}
+            <div class="error-list">
+                <div class="error-description">First 10 errors per category are below. <strong>Full report:</strong> {{ vcf_check_result.get('report_path', '') }}</div>
+                <table>
+                    <tr>
+                        <th>Category</th><th>Error</th>
+                    </tr>
+                    {% for error in critical_list[:10] %}
+                        <tr>
+                            <td><strong>critical error</strong></td><td> {{ error }}</td>
+                        </tr>
+                    {% endfor %}
+                    {% for error in error_list[:10] %}
+                        <tr>
+                            <td><strong>non-critical error</strong></td><td> {{ error }}</td>
+                        </tr>
+                    {% endfor %}
+                </table>
+            </div>
+        {% endif %}
+    {% endfor %}
+{%- endmacro %}
+
+{% macro assembly_check_report(assembly_check_result_per_file, vcf_file_names) -%}
+    {% for file_name in vcf_file_names %}
+        {% set assembly_check_result = assembly_check_result_per_file.get(file_name, {}) %}
+        {% set nb_match = assembly_check_result.get("match", 0) %}
+        {% set nb_total = assembly_check_result.get("total", 0) %}
+        {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %}
+        {% set expand_icon = "" %}
+        {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %}
+            {% set expand_icon = "&#9654;" %}
+            {% set icon = "&#10060;" %}
+            {% set row_class = "report-section fail collapsible" %}
+        {% else %}
+            {% set icon = "&#10004;" %}
+            {% set row_class = "report-section pass" %}
+        {% endif %}
+        <div class='{{ row_class }}'><span class="expand_icon">{{ expand_icon }}</span> {{ icon }} {{ file_name }}: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%)</div>
+        {% set mismatch_list = assembly_check_result.get("mismatch_list") %}
+        {% set error_list = assembly_check_result.get("error_list") %}
+        {% if mismatch_list or error_list %}
+            <div class="error-list">
+            <div class="error-description">First 10 errors per category are below. <strong>Full report:</strong> {{ assembly_check_result.get('report_path', '') }}</div>
+            <table>
+                <tr>
+                    <th>Category</th><th>Error</th>
+                </tr>
+                {% for error in error_list[:10] %}
+                    <tr>
+                        <td><strong>Parsing Error</strong></td><td> {{ error }}</td>
+                    </tr>
+                {% endfor %}
+                {% for error in mismatch_list[:10] %}
+                    <tr>
+                        <td><strong>mismatch error</strong></td><td> {{ error }}</td>
+                    </tr>
+                {% endfor %}
+            </table>
+            </div>
+        {% endif %}
+    {% endfor %}
+{%- endmacro %}
+
+

{eva_sub_cli-0.6.4 → eva_sub_cli-0.6.6}/eva_sub_cli/jinja_templates/html/report.html

@@ -1,6 +1,6 @@
 <!DOCTYPE html>
 {% from 'project_details.html' import project_details_report %}
-{% from 'file_validation.html' import file_validation_report %}
+{% from 'file_validation.html' import assembly_check_report, vcf_check_report %}
 {% from 'sample_name_check.html' import sample_name_check_report %}
 {% from 'fasta_check.html' import fasta_check_report %}
 {% from 'metadata_validation.html' import metadata_validation_report %}
@@ -13,6 +13,13 @@
     <div class='{{ row_class }}'><span class="expand_icon">{{ expand_icon }}</span> {{ icon }} {{ text }}</div>
 {% endmacro %}
 
+{% macro report_crashed_task(task_name) %}
+    {% set icon = "&#10071;" %}   {# exclamation point ❗#}
+    {% set row_class = "report-section info" %}
+    {% set expand_icon = "" %}
+    <div class='{{ row_class }}'><span class="expand_icon">{{ expand_icon }}</span> {{ icon }} {{ task_name }} did not complete successfully. Check the logs for more information.</div>
+{% endmacro %}
+
 <html lang="EN">
 <head>
     <meta charset="UTF-8">
@@ -73,8 +80,10 @@
         For requirements, please refer to the <a href="https://www.ebi.ac.uk/eva/?Submit-Data" target=”_blank”>EVA website</a>.
     </div>
     {% set run_status = validation_results.get('metadata_check', {}).get('run_status', '') %}
-    {% if run_status %}
+    {% if run_status == 'success' %}
         {{ metadata_validation_report(validation_results) }}
+    {% elif run_status == 'crashed' %}
+        {{ report_crashed_task('Metadata check') }}
     {% else %}
         {{ validation_not_run_yet_message() }}
     {% endif %}
@@ -86,14 +95,21 @@
         Checks whether each file is compliant with the <a href="http://samtools.github.io/hts-specs/VCFv4.4.pdf" target=”_blank”>VCF specification</a>.
         Also checks whether the variants' reference alleles match against the reference assembly.
     </div>
+    <h3>Assembly check</h3>
+    {% set run_status = validation_results.get('assembly_check', {}).get('run_status', '') %}
+    {% if run_status == 'success' %}
+        {{ assembly_check_report(validation_results.get('assembly_check', {}), vcf_files) }}
+    {% elif run_status == 'crashed' %}
+        {{ report_crashed_task('Assembly check') }}
+    {% else %}
+        {{ validation_not_run_yet_message() }}
+    {% endif %}
+    <h3>VCF check</h3>
     {% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %}
-    {% if run_status %}
-        {% for file_name in vcf_files %}
-            {% if file_name != "pass"%}
-            <h3>{{ file_name }}</h3>
-            {{ file_validation_report(validation_results, file_name) }}
-            {% endif %}
-        {% endfor %}
+    {% if run_status == 'success' %}
+        {{ vcf_check_report(validation_results.get('vcf_check', {}), vcf_files) }}
+    {% elif run_status == 'crashed' %}
+        {{ report_crashed_task('VCF check') }}
     {% else %}
         {{ validation_not_run_yet_message() }}
     {% endif %}
@@ -105,8 +121,10 @@
         Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names.
     </div>
     {% set run_status = validation_results.get('sample_check', {}).get('run_status', '') %}
-    {% if run_status %}
+    {% if run_status == 'success' %}
         {{ sample_name_check_report(validation_results)}}
+    {% elif run_status == 'crashed' %}
+        {{ report_crashed_task('Sample name concordance') }}
     {% else %}
         {{ validation_not_run_yet_message() }}
 
@@ -120,13 +138,15 @@
         Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file.
     </div>
     {% set run_status = validation_results.get('fasta_check', {}).get('run_status', '') %}
-    {% if run_status %}
+    {% if run_status == 'success' %}
        {% for file_name in fasta_files %}
             {% if file_name != "pass"%}
             <h3>{{ file_name }}</h3>
             {{ fasta_check_report(validation_results, file_name) }}
             {% endif %}
         {% endfor %}
+    {% elif run_status == 'crashed' %}
+        {{ report_crashed_task('INSDC check') }}
     {% else %}
         {{ validation_not_run_yet_message() }}
     {% endif %}

eva_sub_cli-0.6.6/eva_sub_cli/jinja_templates/text/file_validation.txt

@@ -0,0 +1,55 @@
+
+{% macro vcf_check_report(vcf_check_result_per_file, vcf_file_names) -%}
+    {% for file_name in vcf_file_names %}
+        {% set vcf_check_result = vcf_check_result_per_file.get(file_name, {}) %}
+        {% set critical_count = vcf_check_result.get("critical_count", 0) %}
+        {% set error_count = vcf_check_result.get("error_count", 0) %}
+        {% if critical_count > 0 %}
+            {% set icon = "\u274C" %}
+         {% elif error_count > 0 %}
+            {% set icon = "\u26A0" %}
+        {% else %}
+            {% set icon = "\u2714" %}
+        {% endif %}
+		{{ icon }} {{ file_name }}: {{ critical_count }} critical errors, {{ error_count }} non-critical errors
+
+        {% set critical_list = vcf_check_result.get("critical_list") %}
+        {% set error_list = vcf_check_result.get("error_list") %}
+        {% if critical_list or error_list%}
+            First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }}
+            {% for error in critical_list[:10] %}
+                Critical error: {{ error }}
+            {% endfor %}
+            {% for error in error_list[:10] %}
+                Non-critical error: {{ error }}
+            {% endfor %}
+    	{% endif %}
+    {% endfor %}
+{%- endmacro %}
+
+{% macro assembly_check_report(assembly_check_result_per_file, vcf_file_names) -%}
+    {% for file_name in vcf_file_names %}
+        {% set assembly_check_result = assembly_check_result_per_file.get(file_name, {}) %}
+        {% set nb_match = assembly_check_result.get("match", 0) %}
+        {% set nb_total = assembly_check_result.get("total", 0) %}
+        {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %}
+        {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %}
+            {% set icon = "\u274C" %}
+        {% else %}
+            {% set icon = "\u2714" %}
+        {% endif %}
+            {{ icon }} {{ file_name }}: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%)
+
+        {% set mismatch_list = assembly_check_result.get("mismatch_list") %}
+        {% set error_list = assembly_check_result.get("error_list") %}
+        {% if mismatch_list or error_list %}
+            First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }}
+            {% for error in error_list[:10] %}
+                Parsing error: {{ error }}
+            {% endfor %}
+            {% for error in mismatch_list[:10] %}
+                Mismatch error: {{ error }}
+            {% endfor %}
+        {% endif %}
+    {% endfor %}
+{%- endmacro %}

{eva_sub_cli-0.6.4 → eva_sub_cli-0.6.6}/eva_sub_cli/jinja_templates/text/report.txt

@@ -1,5 +1,5 @@
 {% from 'project_details.txt' import project_details_report %}
-{% from 'file_validation.txt' import file_validation_report %}
+{% from 'file_validation.txt' import assembly_check_report, vcf_check_report %}
 {% from 'sample_name_check.txt' import sample_name_check_report %}
 {% from 'fasta_check.txt' import fasta_check_report %}
 {% from 'metadata_validation.txt' import metadata_validation_report %}
@@ -10,6 +10,11 @@
 {{ icon }} {{ text }}
 {%- endmacro %}
 
+{% macro report_crashed_task(task_name) -%}
+{% set icon = "\u2757" %} {# exclamation point ❗#}
+{{ icon }} {{ task_name }} did not complete successfully. Check the logs for more information.
+{%- endmacro %}
+
 VALIDATION REPORT
 eva-sub-cli v{{cli_version}}
 
@@ -29,25 +34,36 @@ Ensures that required fields are present and values are formatted correctly.
 For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data).
 
 {% set run_status = validation_results.get('metadata_check', {}).get('run_status', '') %}
-{% if run_status %}
+{% if run_status == 'success' %}
     {{ metadata_validation_report(validation_results) }}
+{% elif run_status == 'crashed' %}
+    {{ report_crashed_task('Metadata check') }}
 {% else %}
     {{ validation_not_run_yet_message() }}
 {% endif %}
+
 -
 
 VCF VALIDATION RESULTS
 Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf).
 Also checks whether the variants' reference alleles match against the reference assembly.
 
-{% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %}
-{% if run_status %}
-{% for file_name in vcf_files %}
-{% if file_name != "pass"%}
-    {{ file_name }}
-    {{ file_validation_report(validation_results, file_name) }}
+    ASSEMBLY CHECK
+{% set run_status = validation_results.get('assembly_check', {}).get('run_status', '') %}
+{% if run_status == 'success' %}
+    {{ assembly_check_report(validation_results.get('assembly_check', {}), vcf_files) }}
+{% elif run_status == 'crashed' %}
+    {{ report_crashed_task('Assembly check') }}
+{% else %}
+    {{ validation_not_run_yet_message() }}
 {% endif %}
-{% endfor %}
+
+    VCF CHECK
+{% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %}
+{% if run_status == 'success' %}
+    {{ vcf_check_report(validation_results.get('vcf_check', {}), vcf_files)  }}
+{% elif run_status == 'crashed' %}
+    {{ report_crashed_task('VCF check') }}
 {% else %}
     {{ validation_not_run_yet_message() }}
 {% endif %}
@@ -58,8 +74,10 @@ SAMPLE NAME CONCORDANCE CHECK
 Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names.
 
 {% set run_status = validation_results.get('sample_check', {}).get('run_status', '') %}
-{% if run_status %}
+{% if run_status == 'success' %}
     {{ sample_name_check_report(validation_results) }}
+{% elif run_status == 'crashed' %}
+    {{ report_crashed_task('Sample name concordance') }}
 {% else %}
     {{ validation_not_run_yet_message() }}
 {% endif %}
@@ -71,13 +89,16 @@ Checks that the reference sequences in the FASTA file used to call the variants
 Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file.
 
 {% set run_status = validation_results.get('fasta_check', {}).get('run_status', '') %}
-{% if run_status %}
-{% for file_name in fasta_files %}
-{% if file_name != "pass"%}
+{% if run_status == 'success' %}
+    {% for file_name in fasta_files %}
+        {% if file_name != "pass"%}
+    {# Indentation is important as it impact the report's look#}
     {{ file_name }}
     {{ fasta_check_report(validation_results, file_name) }}
-{% endif %}
-{% endfor %}
+        {% endif %}
+    {% endfor %}
+{% elif run_status == 'crashed' %}
+    {{ report_crashed_task('INSDC check') }}
 {% else %}
     {{ validation_not_run_yet_message() }}
 {% endif %}

{eva_sub_cli-0.6.4 → eva_sub_cli-0.6.6}/eva_sub_cli/nextflow/validation.nf

@@ -180,7 +180,6 @@ process check_vcf_valid {
     """
     trap 'if [[ \$? == 1 ]]; then exit 0; fi' EXIT
 
-    rm -rf ${output_dir}/vcf_format
     mkdir -p vcf_format
     $params.executable.vcf_validator -i $vcf -r text -o vcf_format --require-evidence > vcf_format/${vcf}.vcf_format.log 2>&1
     """
@@ -210,7 +209,6 @@ process check_vcf_reference {
     """
     trap 'if [[ \$? == 1 || \$? == 139 ]]; then exit 0; fi' EXIT
 
-    rm -rf ${output_dir}/assembly_check
     mkdir -p assembly_check
     $params.executable.vcf_assembly_checker -i $vcf -f $fasta $report_opt -r summary,text  -o assembly_check --require-genbank > assembly_check/${vcf}.assembly_check.log 2>&1
     """

{eva_sub_cli-0.6.4 → eva_sub_cli-0.6.6}/eva_sub_cli/validators/native_validator.py

@@ -1,8 +1,6 @@
 import os
 import subprocess
 
-from ebi_eva_common_pyutils.logger import logging_config
-
 from eva_sub_cli.validators.validator import Validator, ALL_VALIDATION_TASKS