eva-sub-cli 0.4.9__tar.gz → 0.4.11__tar.gz

eva_sub_cli-0.4.11/.github/workflows/publish_docs.yml

@@ -0,0 +1,50 @@
+name: Deploy documentation with Jekyll / GitHub Pages
+
+on:
+  push:
+    tags:
+      - '**'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./docs
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

{eva_sub_cli-0.4.9 → eva_sub_cli-0.4.11}/.github/workflows/trigger-eva-sub-cli-integration-tests.yml

@@ -2,7 +2,7 @@ name: Trigger Eva Sub Cli Integration Tests
 
 on:
   push:
-    branches: [main, test-trigger-integration-tests]
+    branches: [main]
 
 jobs:
   trigger-integration-tests:
@@ -11,6 +11,6 @@ jobs:
       - name: Trigger eva sub cli integration tests
         uses: peter-evans/repository-dispatch@v2
         with:
-          token: ${{ secrets.TRIGGER_INTEGRATION_TESTS }}
+          token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}
           repository: EBIvariation/eva-integration-tests
           event-type: trigger-eva-sub-cli-tests

eva_sub_cli-0.4.11/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.4
+Name: eva_sub_cli
+Version: 0.4.11
+Summary: EBI EVA - validation and submission command line tool
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Natural Language :: English
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Classifier: Topic :: Communications :: File Sharing
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.3
+Requires-Dist: jinja2==3.*,>=3.0.0
+Requires-Dist: jsonschema==4.*
+Requires-Dist: openpyxl==3.*
+Requires-Dist: pyyaml==6.*
+Requires-Dist: pysam==0.*,>=0.23.3
+Requires-Dist: requests==2.*
+Requires-Dist: packaging==24.*
+Requires-Dist: setuptools-scm
+Dynamic: license-file
+
+# eva-sub-cli
+
+Please view our [documentation website](https://ebivariation.github.io/eva-sub-cli) for help installing and using eva-sub-cli.

eva_sub_cli-0.4.11/README.md

@@ -0,0 +1,3 @@
+# eva-sub-cli
+
+Please view our [documentation website](https://ebivariation.github.io/eva-sub-cli) for help installing and using eva-sub-cli.

{eva_sub_cli-0.4.9 → eva_sub_cli-0.4.11}/docker/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.10
 
-ENV vcf_validator_version=0.10.0
+ENV vcf_validator_version=0.10.2
 ENV NXF_VER=22.10.6
 
 WORKDIR /opt

eva_sub_cli-0.4.11/docs/README.md

@@ -0,0 +1,10 @@
+# About eva-sub-cli
+The eva-sub-cli tool is a command line interface tool for data validation and upload. The tool enables users to perform
+validation and submission to the EVA directly, without requiring interaction with our helpdesk team.
+
+For instructions on how to install eva-sub-cli, see [Installation](installation.md).
+
+For instructions on how to use eva-sub-cli to submit data, see [How to Submit](how_to_submit.md).
+
+## Contact us
+For help or feedback, please get in touch with us at [eva-helpdesk@ebi.ac.uk](mailto:eva-helpdesk@ebi.ac.uk).

eva_sub_cli-0.4.11/docs/_config.yml

@@ -0,0 +1 @@
+remote_theme: just-the-docs/just-the-docs

eva_sub_cli-0.4.11/docs/how_to_submit.md

@@ -0,0 +1,89 @@
+# How to Submit
+
+## 1. File preparation
+You should have **one or more VCF files** to submit, as well as a **completed metadata spreadsheet** following [our 
+template](https://raw.githubusercontent.com/EBIvariation/eva-sub-cli/main/eva_sub_cli/etc/EVA_Submission_template.xlsx).
+More guidance on these files can be found in the [inputs overview](input_file_overview.md) as well as in the metadata 
+template itself.
+
+You will also need the **reference sequences in FASTA format** that you used to generate the VCF files. This is required for
+validation, though it will not be submitted.
+
+## 2. Setting up
+First [**install eva-sub-cli**](installation.md), or check that you have the latest version installed.
+
+You will need an [**ENA Webin account**](https://www.ebi.ac.uk/ena/submit/webin/login) in order to submit, though you
+can run validation without one.
+
+Finally, you need to specify a **submission directory**, which is a specific directory associated with each submission.
+This is where all processing will take place, and where configuration and reports will be saved.
+
+Crucially, the eva-sub-cli tool requires that there be **only one submission per directory** and
+that the submission directory not be reused. Running multiple submissions from a single directory can result in data 
+loss during validation and submission.
+
+## 3. Running eva-sub-cli
+The basic command to run validation and submission is as follows:
+```shell
+eva-sub-cli.py --metadata_xlsx <metadata file> --submission_dir <submission directory> --username <webin username> --password <webin password>
+```
+This will run validation and generate a report with the results. The report can be viewed as HTML or plain text. More 
+description of the validation checks being performed can be found in the [validation overview](validation_overview.md).
+
+**NOTE:** If the validation is successful, this command will automatically submit your data and metadata to EVA. If you 
+don't want this, see [below](#running-only-validation-or-only-submission).
+
+If your files are large, we also encourage you to use [shallow validation](#shallow-validation) before running full
+validation and submission.
+
+## 4. Post submission
+Once you've successfully validated and submitted your data, you will receive an automatic email stating that your data
+has been uploaded. If you don't get this email within a short time, we have very likely **not received your data**, so
+you should get in touch with our [helpdesk](mailto:eva-helpdesk@ebi.ac.uk).
+
+If your submission contains **human genotype data**, then you will also need to provide a signed copy of our
+[consent statement](https://docs.google.com/document/d/1UaRmimAe919IZYIH55mDXMe-4sHsRVqaP4moqx2IYE4) via email.
+For more information please see our [submission FAQs](https://www.ebi.ac.uk/eva/?Help#submissionPanel&link=consent-statement-for-human-genotype-data).
+
+## Other options
+The above steps are the most common, but there are some other options that you can use to tailor its functionality to
+your needs. A full listing is available by running `eva-sub-cli.py -h`.
+
+### Running only validation or only submission
+If you want to run validation without submitting automatically once the validation passes, you can use the `--tasks` 
+option. To run only validation:
+```shell
+eva-sub-cli.py --metadata_xlsx <metadata file> --submission_dir <submission directory> --tasks VALIDATE
+```
+To run only submission - note this will **not submit** if you've not yet validated successfully:
+```shell
+eva-sub-cli.py --metadata_xlsx <metadata file> --submission_dir <submission directory> --tasks SUBMIT --username <webin username> --password <webin password>
+```
+
+### Shallow validation
+If you are working with large VCF files and find that validation takes a very long time, you can add the
+argument `--shallow` to the command, which will validate only the first 10,000 lines in each VCF. Note that running
+shallow validation will **not** be sufficient for actual submission but will allow you to identify most validation concerns without long run time.
+
+### VCF files and reference FASTA
+These can be provided either in the metadata file directly, or on the command line using the `--vcf_files` and
+`--reference_fasta` options. Note that if you are using more than one reference FASTA, you **cannot** use the command 
+line options; you must specify which VCF files use which FASTA files in the metadata.
+
+VCF files can be either uncompressed or compressed using bgzip.
+Other types of compression are not allowed and will result in errors during validation.
+FASTA files must be uncompressed.
+
+### Metadata JSON
+Frequent submitters may be interested in using our [metadata JSON schema](https://github.com/EBIvariation/eva-sub-cli/blob/main/eva_sub_cli/etc/eva_schema.json)
+instead of our spreadsheet template. The metadata requirements are the same regardless of which format you use, you will
+just need to use the `--metadata_json` option instead of the `--metadata_xlsx` option.
+
+### Running using Docker
+If you've installed eva-sub-cli using Docker, make sure that Docker is running in the background before running 
+eva-sub-cli, e.g. by opening Docker Desktop. Additionally, for each eva-sub-cli command, add the command line option 
+`--executor docker`, which will fetch and manage the Docker container for you.
+
+### Using Nextflow
+Under the hood, eva-sub-cli uses Nextflow to run validation, which you can configure on the command line using the 
+`--nextflow_config` option. For more information, please see [here](using_nextflow.md).

eva_sub_cli-0.4.11/docs/input_file_overview.md

@@ -0,0 +1,45 @@
+# Overview of Input Files 
+
+The eva-sub-cli tool requires the following inputs:
+
+- One or several valid VCF files
+- Completed metadata spreadsheet
+- Reference genome in fasta format
+
+VCF files can be either uncompressed or compressed using bgzip.
+Other types of compression are not allowed and will result in errors during validation.
+FASTA files must be uncompressed.
+
+The VCF file must adhere to official VCF specifications, and the metadata spreadsheet provides contextual information about the dataset. In the following sections, we will examine each of these inputs in detail.
+
+# VCF File
+
+A VCF (Variant Call Format) file is a type of file used in bioinformatics to store information about genetic variants. It includes data about the differences (or variants) between a sample's DNA and a reference genome. Typically, generating a VCF file involves several steps: preparing your sample, sequencing the DNA, aligning it to a reference genome, identifying variants, and finally, formatting this information into a VCF file. The overall goal is to systematically capture and record genetic differences in a standardised format. A VCF file consists of two main parts: the header and the body.
+
+Header: The header contains metadata about the file, such as the format version, reference genome information, and descriptions of the data fields. Each line in the header starts with a double ##, except for the last header line which starts with a single #.
+
+```
+##fileformat=VCFv4.2
+##INFO=<ID=DP,Number=1,Type=Integer,Description="Total Depth">
+##FILTER=<ID=PASS,Description="All filters passed">
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+```
+
+Body: The body of the VCF file contains the actual variant data, with each row representing a single variant. The columns in the body are: CHROM, POS, ID, REF, ALT, QUAL, FILTER, INFO, FORMAT, Sample Columns
+
+```
+#CHROM  POS  ID  REF  ALT  QUAL  FILTER  INFO  FORMAT  [SampleIDs...]
+``` 
+
+# Metadata Spreadsheet
+
+The spreadsheet provides comprehensive contextual information about the dataset, ensuring that each submission is accompanied by detailed descriptions that facilitate proper understanding and use of the data. Key elements included in the metadata spreadsheet are analysis and project information, sample information, sequencing methodologies, experimental details.
+The spreadsheet is composed of editable tabs that will receive the metadata and non-editable helper tabs that provide detailed description of each column.
+
+| WORKSHEET         | EXPLANATION                                                                                                                                                                                                                                                                                                                                                       |
+|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Submitter Details | This sheet captures the details of the submitter                                                                                                                                                                                                                                                                                                                  |
+| Project           | The objective of this sheet is to gather general information about the Project including submitter, submitting centre, collaborators, project title, description and publications.                                                                                                                                                                                |
+| Sample            | Projects consist of analyses that are run on samples. We accept sample information in the form of BioSample, ENA or EGA accession(s). We also accept BioSamples sampleset accessions. If your samples are not yet accessioned, and are therefore novel, please use the "Novel sample(s)" sections of the Sample(s) worksheet to have them registered at BioSample |
+| Analysis          | For EVA, each analysis is one vcf file, plus an unlimited number of ancillary files. This sheet allows EVA to link vcf files to a project and to other EVA analyses. Additionally, this worksheet contains experimental meta-data detailing the methodology of each analysis. Important to note; one project can have multiple associated analyses                |
+| Files             | Filenames and associated checking data associated with this EVA submission should be entered into this worksheet. Each file should be linked to exactly one analysis.                                                                                                                                                                                             |

eva_sub_cli-0.4.11/docs/installation.md

@@ -0,0 +1,49 @@
+# Installation
+
+There are currently three ways to install and run eva-sub-cli:
+- Using conda
+- From source using Docker
+- From source natively (i.e. installing dependencies yourself)
+
+We encourage users to install the most recent version, as we are constantly improving the tool. You can check that you
+have the latest version installed by running `eva-sub-cli.py --version` and comparing against the
+[release page](https://github.com/EBIvariation/eva-sub-cli/releases).
+
+## 1. Conda
+
+The most straightforward way to install eva-sub-cli and its dependencies is through conda.
+For instance, the following commands install eva-sub-cli in a new environment called `eva`, activate the environment, and print
+the help message:
+```bash
+conda create -n eva -c conda-forge -c bioconda eva-sub-cli
+conda activate eva
+eva-sub-cli.py --help
+````
+
+To upgrade to the newest version, run `conda update eva-sub-cli`.
+
+## 2. From source using Docker
+
+Docker provides an easy way to run eva-sub-cli without installing dependencies separately.
+This method requires just Python 3.8+ and [Docker](https://docs.docker.com/engine/install/) to be installed.
+Then you can install the most recent version from [PyPI](https://pypi.org/project/eva-sub-cli/) in a virtual environment:
+```bash
+pip install eva-sub-cli
+```
+
+To verify that the cli tool is installed correctly, run the following command, and you should see the help message displayed: 
+```bash
+eva-sub-cli.py --help
+```
+
+To upgrade to the newest version, run `pip install --upgrade eva-sub-cli`.
+
+## 3. From source natively
+
+This installation method requires the following:
+* Python 3.8+
+* [Nextflow](https://www.nextflow.io/docs/latest/getstarted.html) 21.10+
+* [biovalidator](https://github.com/elixir-europe/biovalidator) 2.1.0+
+* [vcf-validator](https://github.com/EBIvariation/vcf-validator) 0.9.7+
+
+Install each of these and ensure they are included in your PATH. Then install the latest eva-sub-cli release as previously described.

eva_sub_cli-0.4.11/docs/using_nextflow.md

@@ -0,0 +1,48 @@
+# Using Nextflow for Parallelization  
+
+Nextflow is a common workflow management system that helps orchestrate tasks and interface with the execution engine (like HPC or cloud). When running natively (i.e. not using Docker), eva-sub-cli will use Nextflow to run all the validation steps. In this section we'll see how it can be parameterised to work with your compute infrastructure.
+
+When no options are provided, Nextflow will run as many tasks as there are available CPUs on the machine executing it. To modify how many tasks can start and how Nextflow will process each one, you can provide a Nextflow configuration file in several ways.
+
+From the command line you can use `--nextflow_config <path>` to specify the Nextflow config file you want to apply. The configuration can also be picked up from other places directly by Nextflow. Please refer to [the Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more details.
+
+## Basic Nextflow configuration
+
+There are many options to configure Nextflow so we will not provide them all. Please refer to [the documentation](https://www.nextflow.io/docs/latest/reference/config.html) for advanced features.
+Below is a very basic Nextflow configuration file that will request 2 cpus for each process, essentially limiting the number of process to half the number of available CPUs 
+```
+process {
+    executor="local"
+    cpus=2
+}
+```
+In this configuration, all the process will be running on the same machine where eva-sub-cli was started as described in the schema below.
+```
+(Local machine)
+eva-sub-cli
+  |_ nextflow
+      |_ task1
+      |_ task2
+```
+
+## Basic Nextflow configuration for HPC use
+
+If you have access to High Performance Compute (HPC) environment, Nextflow supports the main resource managers such as [SLURM](https://www.nextflow.io/docs/latest/executor.html#slurm), [SGE](https://www.nextflow.io/docs/latest/executor.html#sge), [LSF](https://www.nextflow.io/docs/latest/executor.html#lsf) and others.
+In the configuration below, we're assuming that you are using SLURM. It would work similarly with other resource managers.
+```
+process {
+    executor="slurm"
+    queue="my_production_queue"
+}
+```
+
+In this configuration, the subtasks will be performed in other machines as specified by your SLURM resource manager as described in the schema below.
+```
+(Local machine)
+eva-sub-cli
+  |_ nextflow
+(Other compute node)
+task1
+(Other compute node)
+task2
+```

eva_sub_cli-0.4.9/docs/Getting_Started_with_eva_sub_cli.md → eva_sub_cli-0.4.11/docs/validation_overview.md

@@ -1,51 +1,4 @@
-# Comprehensive overview of the required inputs for eva-sub-cli tool 
-
-The eva-sub-cli tool requires the following inputs:
-
-- One or several valid VCF files
-- Completed metadata spreadsheet
-- Reference genome in fasta format
-
-The VCF file must adhere to official VCF specifications, and the metadata spreadsheet provides contextual information about the dataset. In the following sections, we will examine each of these inputs in detail.
-
-# VCF File
-
-A VCF (Variant Call Format) file is a type of file used in bioinformatics to store information about genetic variants. It includes data about the differences (or variants) between a sample's DNA and a reference genome. Typically, generating a VCF file involves several steps: preparing your sample, sequencing the DNA, aligning it to a reference genome, identifying variants, and finally, formatting this information into a VCF file. The overall goal is to systematically capture and record genetic differences in a standardised format. A VCF file consists of two main parts: the header and the body.
-
-Header: The header contains metadata about the file, such as the format version, reference genome information, and descriptions of the data fields. Each line in the header starts with a double ##, except for the last header line which starts with a single #.
-
-File format version
-
-```
-##fileformat=VCFv4.2
-##INFO=<ID=DP,Number=1,Type=Integer,Description="Total Depth">
-##FILTER=<ID=PASS,Description="All filters passed">
-##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
-
-```
-
-Body: The body of the VCF file contains the actual variant data, with each row representing a single variant. The columns in the body are : CHROM, POS, ID, REF, ALT, QUAL, FILTER, INFO, FORMAT, Sample Columns
-
-```
-#CHROM  POS  ID  REF  ALT  QUAL  FILTER  INFO  FORMAT  [SampleIDs...]
-``` 
-Here's a small example to illustrate the structure of a VCF file: Example VCF file archived at EVA to be inserted 
-
-# Metadata Spreadsheet
-
-The spreadsheet provides comprehensive contextual information about the dataset, ensuring that each submission is accompanied by detailed descriptions that facilitate proper understanding and use of the data. Key elements included in the metadata spreadsheet are analysis and project information, sample information, sequencing methodologies, experimental details. 
-
-
-| WORKSHEET           | EXPLANATION          |
-| -----------------   | -------------------- |
-|  Submitter Details   | This sheet captures the details of the submitter|
-| Project           | The objective of this sheet is to gather general information about the Project including submitter, submitting centre, collaborators, project title, description and publications. |
-| Sample            | Projects consist of analyses that are run on samples. We accept sample information in the form of BioSample, ENA or EGA accession(s). We also accept BioSamples sampleset accessions. If your samples are not yet accessioned, and are therefore novel, please use the "Novel sample(s)" sections of the Sample(s) worksheet to have them registered at BioSample |
-| Analysis          | For EVA, each analysis is one vcf file, plus an unlimited number of ancillary files. This sheet allows EVA to link vcf files to a project and to other EVA analyses. Additionally, this worksheet contains experimental meta-data detailing the methodology of each analysis. Important to note; one project can have multiple associated analyses          |
-| Files             | Filenames and associated checking data associated with this EVA submission should be entered into this worksheet. Each file should be linked to exactly one analysis.            |
-
-
-# Validation checks 
+# Overview of Validation Checks 
 
 The CLI tool performs the following validation checks and generates corresponding reports:
 
@@ -56,7 +9,7 @@ The CLI tool performs the following validation checks and generates correspondin
 
 In the following sections, we will examine each of these checks in detail, starting with the Metadata check.
 
-# Metadata check
+## Metadata check
 
 Once the user passes the metadata spreadsheet for validation checks, the eva-sub-cli tool verifies that all mandatory columns, marked in bold in the spreadsheet, are filled in. This data is crucial for further validation processes, such as retrieving the INDSC accession of the reference genome used to call the variants, and for sample and project metadata. If any mandatory columns or sheets are missing, the CLI tool will raise errors.
 
@@ -75,7 +28,7 @@ Common Errors Seen with Metadata Checks:
 - Tax ID and the scientific name of the organism do not match.
 - Collection data and geographic location of the samples are not filled if the samples being submitted are novel.
 
-# VCF Checks
+## VCF Checks
 
 Ensuring data consistency upon submission is crucial for interoperability and supporting cross-study comparative genomics. Before accepting a VCF submission, the cli tool verifies that the submitted information adheres to the official VCF specifications. Additionally, submitted variants must be supported by either experimentally determined sample genotypes or population allele frequencies.
 
@@ -95,7 +48,7 @@ Common Errors Seen with VCF Checks:
 - VCF uses non-GCA contig alias
 - The fields used do not conform to the official VCF specifications 
 
-# Assembly Check
+## Assembly Check
 
 The EVA requires that all variants be submitted with an asserted position on an INSDC sequence. This means that the reference allele for every variant must match a position in a sequence that has been accessioned in either the GenBank or ENA database. Aligning all submitted data with INSDC sequences enables integration with other EMBL-EBI resources, including Ensembl, and is crucial for maintaining standardisation at the EVA. Therefore, all sequence identifiers in your VCF must match those in the reference FASTA file.
 
@@ -110,7 +63,7 @@ Key points to note before validating your data with the eva-sub-cli Tool:
 - Contigs used do not exist in the assembly report of the reference genome
 - Major Allele Used as REF Allele: This typically occurs when a specific version of Plink or Tassel is used to create VCF files, causing the tool to use the major allele as the reference allele. In such cases, submitters should use the GCA FASTA sequence to create corrected files.
 
-# Sample Name Concordance Check
+## Sample Name Concordance Check
 
 The sample name concordance check ensures that the sample names in the metadata spreadsheet match those in the VCF file. This is achieved by cross-checking the 'Sample name in VCF' column in the spreadsheet with the sample names registered in the VCF file. Any discrepancies must be addressed by the submitter when the CLI tool generates a report of the  mismatches found.
 
@@ -124,5 +77,3 @@ Common errors seen with sample concordance checks:
 - Link between “Sample” and “File” provided via the Analysis alias is not correctly defined in the metadata which causes the sample name concordance check to fail.
 - Extra white spaces in the sample names can lead to mismatches.
 - Case sensitivity issues between the sample names in the VCF file and the metadata spreadsheet.
-
-

{eva_sub_cli-0.4.9 → eva_sub_cli-0.4.11}/eva_sub_cli/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.9'
-__version_tuple__ = version_tuple = (0, 4, 9)
+__version__ = version = '0.4.11'
+__version_tuple__ = version_tuple = (0, 4, 11)

{eva_sub_cli-0.4.9 → eva_sub_cli-0.4.11}/eva_sub_cli/etc/eva_schema.json

@@ -168,6 +168,14 @@
             "type": "string",
             "description": "Reference against which the analysis was performed. ENA or NCBI accession (starting with GCA) accepted."
           },
+          "evidenceType": {
+            "type": "string",
+            "description": "The type of evidence (allele frequencies or genotypes) provided in the VCF files of the analysis",
+            "enum": [
+              "genotype",
+              "allele_frequency"
+            ]
+          },
           "referenceFasta": {
             "type": "string",
             "description": "The path to the reference in fasta format that was used to create the VCF. (This is a local path)"
@@ -366,7 +374,7 @@
     "projectAccession": {
       "type": "string",
       "description": "ENA project accession",
-      "pattern": "^PRJ(EB|NA)\\d+$"
+      "pattern": "^PRJ(E|D|N)[A-Z][0-9]+$"
     }
   }
 }

eva_sub_cli-0.4.11/eva_sub_cli/executables/evidence_type_checker.py

@@ -0,0 +1,51 @@
+import argparse
+
+import yaml
+from ebi_eva_common_pyutils.logger import logging_config
+
+from eva_sub_cli.file_utils import detect_vcf_evidence_type
+from eva_sub_cli.metadata import EvaMetadataJson
+
+logger = logging_config.get_logger(__name__)
+
+
+def check_evidence_type(metadata_json, output_yaml):
+    metadata = EvaMetadataJson(metadata_json)
+    results_per_analysis_alias = {}
+
+    for analysis_alias, vcf_file_set in metadata.files_per_analysis.items():
+        results_per_analysis_alias[analysis_alias] = {
+            'evidence_type': None,
+            'errors': None
+        }
+        vcf_files = list(vcf_file_set)
+        evidence_types_for_vcf_files = [detect_vcf_evidence_type(vcf_file) for vcf_file in vcf_files]
+
+        if len(set(evidence_types_for_vcf_files)) == 1 and None not in evidence_types_for_vcf_files:
+            results_per_analysis_alias[analysis_alias]['evidence_type'] = set(evidence_types_for_vcf_files).pop()
+        elif None in evidence_types_for_vcf_files:
+            indices = [i for i, x in enumerate(evidence_types_for_vcf_files) if x is None]
+            results_per_analysis_alias[analysis_alias][
+                'errors'] = f'VCF file evidence type could not be determined: {", ".join([vcf_files[i] for i in indices])}'
+        else:
+            results_per_analysis_alias[analysis_alias][
+                'errors'] = f'Multiple evidence types found: {", ".join(set(evidence_types_for_vcf_files))}'
+
+    write_result_yaml(output_yaml, results_per_analysis_alias)
+
+
+def write_result_yaml(output_yaml, results_per_analysis_alias):
+    with open(output_yaml, 'w') as open_yaml:
+        yaml.safe_dump(data=results_per_analysis_alias, stream=open_yaml, default_flow_style=False)
+
+
+def main():
+    arg_parser = argparse.ArgumentParser(description='Check the evidence type in the VCF files for all analysis')
+    arg_parser.add_argument('--metadata_json', required=True, dest='metadata_json',
+                            help='EVA metadata json file')
+    arg_parser.add_argument('--output_yaml', required=True, dest='output_yaml',
+                            help='Path to the location of the results')
+
+    args = arg_parser.parse_args()
+    logging_config.add_stdout_handler()
+    check_evidence_type(args.metadata_json, args.output_yaml)

{eva_sub_cli-0.4.9 → eva_sub_cli-0.4.11}/eva_sub_cli/executables/samples_checker.py

@@ -5,7 +5,7 @@ from ebi_eva_common_pyutils.logger import logging_config
 
 import yaml
 
-from eva_sub_cli.file_utils import open_gzip_if_required
+from eva_sub_cli.file_utils import open_gzip_if_required, detect_vcf_evidence_type
 from eva_sub_cli.metadata import EvaMetadataJson
 
 logger = logging_config.get_logger(__name__)
@@ -64,20 +64,41 @@ def compare_all_analysis(metadata, files_per_analysis):
             file_path: get_samples_from_vcf(file_path)
             for file_path in files_per_analysis.get(analysis_alias, [])
         }
-        (
-            has_difference, more_per_submitted_files_metadata,
-            more_submitted_files_metadata, more_metadata_submitted_files
-        ) = compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_file)
-        overall_differences = overall_differences or has_difference
-        results_per_analysis_alias[analysis_alias] = {
-            'difference': has_difference,
-            'more_per_submitted_files_metadata': more_per_submitted_files_metadata,
-            'more_submitted_files_metadata': more_submitted_files_metadata,
-            'more_metadata_submitted_files': more_metadata_submitted_files
-        }
+
+        if need_to_check_samples(sample_name_per_file):
+            (
+                has_difference, more_per_submitted_files_metadata,
+                more_submitted_files_metadata, more_metadata_submitted_files
+            ) = compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_file)
+            overall_differences = overall_differences or has_difference
+            results_per_analysis_alias[analysis_alias] = {
+                'difference': has_difference,
+                'more_per_submitted_files_metadata': more_per_submitted_files_metadata,
+                'more_submitted_files_metadata': more_submitted_files_metadata,
+                'more_metadata_submitted_files': more_metadata_submitted_files
+            }
+        else:
+            results_per_analysis_alias[analysis_alias] = {
+                'difference': False,
+                'more_per_submitted_files_metadata': [],
+                'more_submitted_files_metadata': [],
+                'more_metadata_submitted_files': []
+            }
+
+
     return overall_differences, results_per_analysis_alias
 
 
+def need_to_check_samples(sample_name_per_file):
+    no_samples_in_vcf = all(len(v) == 0 for v in sample_name_per_file.values())
+    if no_samples_in_vcf:
+        evidence_types_for_vcf_files = [detect_vcf_evidence_type(vcf_file) for vcf_file in sample_name_per_file.keys()]
+        if set(evidence_types_for_vcf_files) == {'allele_frequency'}:
+            return False
+
+    return True
+
+
 def associate_vcf_path_with_analysis(metadata, vcf_files):
     """
     Match the files names associated with analysis provided in the metadata with the file path given on the command