eva-sub-cli 0.4.13__tar.gz → 0.5.0__tar.gz

{eva_sub_cli-0.4.13 → eva_sub_cli-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.4.13
+Version: 0.5.0
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console

{eva_sub_cli-0.4.13 → eva_sub_cli-0.5.0}/docker/Dockerfile

@@ -1,7 +1,7 @@
 FROM python:3.10
 
 ENV vcf_validator_version=0.10.2
-ENV NXF_VER=22.10.6
+ENV NXF_VER=23.10.0
 
 WORKDIR /opt
 

eva_sub_cli-0.5.0/docs/installation.md

@@ -0,0 +1,70 @@
+# Installation
+
+There are two main ways to install eva-sub-cli, either through [conda](#1-conda) or [pip](#2-pip).
+Conda is the most straightforward and recommended method, as it will install all dependencies for you.
+
+Otherwise you can install using pip, in which case you will need to handle non-Python dependencies separately.
+You can do this either using Docker or natively (i.e. installing by yourself).
+Note that pip using Docker is currently the only installation method available for Windows users.
+
+We encourage users to install the most recent version, as we are constantly improving the tool. You can check that you
+have the latest version installed by running `eva-sub-cli.py --version` and comparing against the
+[release page](https://github.com/EBIvariation/eva-sub-cli/releases).
+
+All installation methods require Python 3.8 or above.
+
+## 1. Conda
+
+View our [video tutorial on conda installation](https://embl-ebi.cloud.panopto.eu/Panopto/Pages/Viewer.aspx?id=aa82710e-4401-4074-b6b0-b346016bb14e).
+
+The most straightforward way to install eva-sub-cli and its dependencies is through conda.
+For instance, the following commands install eva-sub-cli in a new environment called `eva`, activate the environment, and print
+the help message:
+```bash
+conda create -n eva -c conda-forge -c bioconda eva-sub-cli
+conda activate eva
+eva-sub-cli.py --help
+````
+
+To upgrade to the newest version, run `conda update eva-sub-cli`.
+
+## 2. Pip
+
+View our [video tutorial on pip installation](https://embl-ebi.cloud.panopto.eu/Panopto/Pages/Viewer.aspx?id=2e6828ed-31c2-4621-8eb5-b34701484aae).
+
+You can also install eva-sub-cli using pip. 
+The following commands will create a new virtual environment called `eva`, activate the environment, and install eva-sub-cli:
+```bash
+python3 -m venv eva
+source ./eva/bin/activate
+pip install eva-sub-cli
+```
+
+You can verify the installation by printing the help message:
+```bash
+eva-sub-cli.py --help
+```
+
+For external (non-Python) dependencies, you will need to install them either [via Docker](#docker) or [natively](#native).
+These methods are described below.
+
+To upgrade to the newest version of the tool, run `pip install --upgrade eva-sub-cli`.
+
+### Docker
+
+Docker provides an easy way to run eva-sub-cli without installing dependencies separately.
+For this method, all you need is for [Docker](https://docs.docker.com/engine/install/) to be installed and running in the background.
+
+When you run eva-sub-cli, you will need to [specify execution using Docker](how_to_submit.md#running-using-docker).
+The tool will then automatically run using a Docker container with all dependencies pre-installed.
+
+### Native
+
+For those requiring more flexibility or working on systems where Docker is not available, you can also install the 
+external dependencies on your own. These are:
+* [Nextflow](https://www.nextflow.io/docs/latest/getstarted.html) 21.10+
+* [biovalidator](https://github.com/elixir-europe/biovalidator) 2.1.0+
+* [vcf-validator](https://github.com/EBIvariation/vcf-validator) 0.9.7+
+
+Install each of these and ensure they are included in your PATH.
+This will allow eva-sub-cli to be executed natively, i.e. directly on your machine.

{eva_sub_cli-0.4.13 → eva_sub_cli-0.5.0}/eva_sub_cli/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.4.13'
-__version_tuple__ = version_tuple = (0, 4, 13)
+__version__ = version = '0.5.0'
+__version_tuple__ = version_tuple = (0, 5, 0)
 
-__commit_id__ = commit_id = 'gc6e52eaa9'
+__commit_id__ = commit_id = 'g1b663fd41'

{eva_sub_cli-0.4.13 → eva_sub_cli-0.5.0}/eva_sub_cli/etc/eva_schema.json

@@ -64,66 +64,81 @@
       }
     },
     "project": {
-      "type": "object",
-      "description": "General information about the Project",
-      "required": [
-        "title",
-        "description",
-        "taxId",
-        "centre"
-      ],
-      "properties": {
-        "title": {
-          "type": "string",
-          "description": "Title of the project",
-          "maxLength": 500
-        },
-        "description": {
-          "type": "string",
-          "description": "Description of the project",
-          "maxLength": 5000
-        },
-        "centre": {
-          "type": "string",
-          "description": "Centre, institute or company submitting the project"
-        },
-        "taxId": {
-          "type": "number",
-          "description": "Taxonomy ID of the species associated with the project (found here: http://www.ncbi.nlm.nih.gov/taxonomy)"
-        },
-        "publications": {
-          "type": "array",
-          "items": true
-        },
-        "parentProject": {
-          "$ref": "#/definitions/projectAccession"
-        },
-        "childProjects": {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/projectAccession"
-          }
-        },
-        "peerProjects": {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/projectAccession"
+      "oneOf": [
+        {
+          "type": "object",
+          "required": [
+            "projectAccession"
+          ],
+          "properties": {
+            "projectAccession": {
+              "$ref": "#/definitions/projectAccession"
+            }
           }
         },
-        "links": {
-          "type": "array",
-          "items": {
-            "type": "string"
+        {
+          "type": "object",
+          "description": "General information about the Project",
+          "required": [
+            "title",
+            "description",
+            "taxId",
+            "centre"
+          ],
+          "properties": {
+            "title": {
+              "type": "string",
+              "description": "Title of the project",
+              "maxLength": 500
+            },
+            "description": {
+              "type": "string",
+              "description": "Description of the project",
+              "maxLength": 5000
+            },
+            "centre": {
+              "type": "string",
+              "description": "Centre, institute or company submitting the project"
+            },
+            "taxId": {
+              "type": "number",
+              "description": "Taxonomy ID of the species associated with the project (found here: http://www.ncbi.nlm.nih.gov/taxonomy)"
+            },
+            "publications": {
+              "type": "array",
+              "items": true
+            },
+            "parentProject": {
+              "$ref": "#/definitions/projectAccession"
+            },
+            "childProjects": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/projectAccession"
+              }
+            },
+            "peerProjects": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/projectAccession"
+              }
+            },
+            "links": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "holdDate": {
+              "type": "string",
+              "format": "date",
+              "examples": [
+                "2018-11-13"
+              ]
+            }
           }
-        },
-        "holdDate": {
-          "type": "string",
-          "format": "date",
-          "examples": [
-            "2018-11-13"
-          ]
         }
-      }
+      ]
     },
     "analysis": {
       "type": "array",
@@ -324,18 +339,7 @@
             "description": "BioSample accession of pre-existing sample. Only use for preregistered samples"
           },
           "bioSampleObject": {
-            "type": "object",
-            "anyOf": [
-              {
-                "$ref": "eva-biosamples.json"
-              },
-              {
-                "$ref": "https://www.ebi.ac.uk/biosamples/schema-store/registry/schemas/ERC000011"
-              },
-              {
-                "$ref": "https://www.ebi.ac.uk/biosamples/schemas/certification/plant-miappe.json"
-              }
-            ]
+            "$ref": "https://www.ebi.ac.uk/biosamples/schema-store/registry/schemas/ERC000011"
           }
         }
       }

eva_sub_cli-0.5.0/eva_sub_cli/etc/spreadsheet2json_conf.yaml

@@ -0,0 +1,139 @@
+#Configuration for defining which fields in which worksheets to be parsed in an Excel file
+#
+#At the top level, "worksheets" defines a list of worksheets (titles) to be parsed.
+#Then for each worksheet, you define which fields to be parsed from this worksheet.
+#These fields are either "required", "optional" and/or header_row. "required" fields must be found
+#in the worksheet so that this worksheet is valid. Otherwise this worksheet is not
+#valid. The field names must match those values in the header row in the worksheet.
+#
+#This is a YAML file
+
+# List of worksheets to be parsed.
+worksheets:
+  Submitter Details: submitterDetails
+  Project: project
+  Analysis: analysis
+  Sample: sample
+  Files: files
+
+Submitter Details:
+  required:
+    Last Name: lastName
+    First Name: firstName
+    Email Address: email
+    Laboratory: laboratory
+    Center: centre
+  optional:
+    Address: address
+
+# List of field for Projects
+Project:
+  header_row: 3
+  optional:
+    Project Accession: projectAccession
+    Project Title: title
+    Description: description
+    Center: centre
+    Taxonomy ID: taxId
+    Publication(s): publications
+    Parent Project: parentProject
+    Child Project(s): childProjects
+    Peer Project(s): peerProjects
+    Link(s): links
+    Hold Date: holdDate
+    Collaborator(s): collaborators
+    Strain: strain
+    Breed: breed
+    Broker: broker
+  cast:
+    Publication(s): list
+    Child Project(s): list
+    Peer Project(s): list
+    Link(s): list
+    Hold Date: date
+    Taxonomy ID: integer
+
+Analysis:
+  required:
+    Analysis Title: analysisTitle
+    Analysis Alias: analysisAlias
+    Description: description
+    Experiment Type: experimentType
+    Reference: referenceGenome
+  optional:
+    Reference Fasta Path: referenceFasta
+    Platform: platform
+    Software: software
+    Pipeline Description: pipelineDescriptions
+    Imputation: imputation
+    Phasing: phasing
+    Centre: centre
+    Date: date
+    Link(s): links
+    Run Accession(s): runAccessions
+  cast:
+    Run Accession(s): list
+    Imputation: boolean
+    Phasing: boolean
+    Date: date
+    Software: list
+
+Sample:
+  header_row: 3
+  required:
+    Analysis Alias: analysisAlias
+    Sample Name in VCF: sampleInVCF
+  # Sample property names taken from ERC000011 where present
+  # See https://www.ebi.ac.uk/biosamples/schema-store/registry/schemas/ERC000011
+  optional:
+    Sample Accession: bioSampleAccession
+    BioSample Name: bioSampleName
+    Title: title
+    Description: description
+    Unique Name: uniqueName
+    Prefix: prefix
+    Subject: subject
+    Derived From: derivedFrom
+    Taxonomy ID: taxId
+    Scientific Name: scientific name
+    Common Name: commonName
+    mating_type: mating_type
+    sex: sex
+    population: population
+    cell_type: cell_type
+    dev_stage: dev_stage
+    germline: germline
+    tissue_lib: tissue_lib
+    tissue_type: tissue_type
+    bio_material: bio_material
+    culture_collection: culture_collection
+    specimen_voucher: specimen_voucher
+    collected_by: collected_by
+    collection_date: collection date
+    geographic location (country and/or sea): geographic location (country and/or sea)
+    geographic location (region and locality): geographic location (region and locality)
+    host: host
+    identified_by: identified_by
+    isolation_source: isolation_source
+    lat_lon: lat_lon
+    lab_host: lab_host
+    environmental_sample: environmental_sample
+    cultivar: cultivar
+    ecotype: ecotype
+    isolate: isolate
+    strain: strain
+    sub_species: sub_species
+    variety: variety
+    sub_strain: sub_strain
+    cell_line: cell_line
+    serotype: serotype
+    serovar: serovar
+  cast:
+    collection_date: date
+    Taxonomy ID: integer
+
+
+Files:
+  required:
+    Analysis Alias: analysisAlias
+    File Name: fileName

eva_sub_cli-0.4.13/eva_sub_cli/etc/spreadsheet2json_conf.yaml → eva_sub_cli-0.5.0/eva_sub_cli/etc/spreadsheet2json_conf_V2.yaml

@@ -50,6 +50,8 @@ Project:
     Peer Project(s): list
     Link(s): list
     Hold Date: date
+    Taxonomy ID: integer
+
 
 Analysis:
   required:
@@ -127,12 +129,12 @@ Sample:
     serotype: serotype
     serovar: serovar
   cast:
-    Sample ID: string
-    BioSample Name: string
     collection_date: date
+    Taxonomy ID: integer
+
 
 
 Files:
   required:
     Analysis Alias: analysisAlias
-    File Name: fileName
+    File Name: fileName

{eva_sub_cli-0.4.13 → eva_sub_cli-0.5.0}/eva_sub_cli/executables/cli.py

@@ -22,24 +22,47 @@ from eva_sub_cli.file_utils import is_submission_dir_writable, DirLockError, Dir
 
 
 def validate_command_line_arguments(args, argparser):
+    fail = False
     if (args.vcf_files and not args.reference_fasta) or (not args.vcf_files and args.reference_fasta):
         print("When using --vcf_files and --reference_fasta, both need to be specified")
-        argparser.print_usage()
-        sys.exit(1)
+        fail = True
+
+    if args.vcf_files:
+        for vcf_file in args.vcf_files:
+            if not os.path.isfile(vcf_file):
+                print(f"VCF file {vcf_file} is not a file")
+                fail = True
+
+    if args.reference_fasta:
+        if not os.path.isfile(args.reference_fasta):
+            print(f"Fasta file {args.reference_fasta} is not a file")
+            fail = True
+
+    if args.metadata_xlsx:
+        if not os.path.isfile(args.metadata_xlsx):
+            print(f"Spreadsheet file {args.metadata_xlsx} is not a file")
+            fail = True
+
+    if args.metadata_json:
+        if not os.path.isfile(args.metadata_json):
+            print(f"JSON file {args.metadata_json} is not a file")
+            fail = True
 
     if SUBMIT in args.tasks and (
             not (args.username or os.environ.get('ENAWEBINACCOUNT')) or
             not (args.password or os.environ.get('ENAWEBINPASSWORD'))):
         print("To submit your data, you need to provide a Webin username and password")
-        argparser.print_usage()
-        sys.exit(1)
+        fail = True
 
     if not is_submission_dir_writable(args.submission_dir):
         print(f"'{args.submission_dir}' does not have write permissions or is not a directory.")
-        sys.exit(1)
+        fail = True
 
     if args.nextflow_config and not os.path.isfile(args.nextflow_config):
         print(f"'{args.nextflow_config}' is not a file or does not exist.")
+        fail = True
+
+    if fail:
         argparser.print_usage()
         sys.exit(1)
 
@@ -128,4 +151,7 @@ def main():
     except MetadataTemplateVersionNotFoundException as mte:
         print(mte)
         exit_status = 70
+    except Exception as ex:
+        print(ex)
+        exit_status = 71
     return exit_status

{eva_sub_cli-0.4.13 → eva_sub_cli-0.5.0}/eva_sub_cli/executables/xlsx2json.py

@@ -103,9 +103,7 @@ class XlsxParser:
     @staticmethod
     def cast_value(value, type_name):
         # Do not cast None values
-        if type_name and value is not None:
-            if type_name == 'string':
-                return str(value)
+        if value is not None:
             if type_name == 'boolean':
                 return str(value).lower() in ['true', '1', "'1'", 't', 'y', 'yes']
             if type_name == 'list':
@@ -114,6 +112,10 @@ class XlsxParser:
                 return [v for v in split_values if v]
             if type_name == 'date':
                 return XlsxParser.serialize(value)
+            if type_name == 'integer':
+                return int(value)
+            # Default any value is cast to string
+            return str(value)
         return value
 
     @staticmethod
@@ -179,15 +181,20 @@ class XlsxParser:
         return rows
 
     def get_project_json_data(self):
+        json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][PROJECT]
+
         project_data = self.get_rows()
         if len(project_data) > 1:
             logger.warning(f"{PROJECT} worksheet expects a single row of info but more than one found in the file. "
                            f"Only the first row's data will be taken into consideration")
+        elif len(project_data) == 0:
+            logger.error(f"{PROJECT} worksheet expects a single row of info but no info is found in the file.")
+            self.add_error(f"{PROJECT} worksheet expects a single row of info but no info is found in the file.")
+            return {json_key: {}}
 
         first_row = project_data[0]
         first_row.pop('row_num')
 
-        json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][PROJECT]
         json_value = {self.translate_header(PROJECT, k): v for k, v in first_row.items() if v is not None}
         return {json_key: json_value}
 

{eva_sub_cli-0.4.13 → eva_sub_cli-0.5.0}/eva_sub_cli/jinja_templates/html/file_validation.html

@@ -20,7 +20,7 @@
         {% set row_class = "report-section fail collapsible" %}
     {% elif error_count > 0 %}
         {% set expand_icon = "▶" %}
-        {% set icon = "❌" %}
+        {% set icon = "❗" %}
         {% set row_class = "report-section warn collapsible" %}
     {% else %}
         {% set icon = "✔" %}

{eva_sub_cli-0.4.13 → eva_sub_cli-0.5.0}/eva_sub_cli/jinja_templates/text/file_validation.txt

@@ -13,8 +13,10 @@
 {% macro vcf_check(vcf_check_result) %}
     {% set critical_count = vcf_check_result.get("critical_count", 0) %}
     {% set error_count = vcf_check_result.get("error_count", 0) %}
-    {% if critical_count > 0 or error_count > 0 %}
+    {% if critical_count > 0 %}
         {% set icon = "\u274C" %}
+     {% elif error_count > 0 %}
+        {% set icon = "\u26A0" %}
     {% else %}
         {% set icon = "\u2714" %}
     {% endif %}