eva-sub-cli 0.4.4__py3-none-any.whl

eva_sub_cli/etc/eva_schema.json

@@ -0,0 +1,369 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "version": "1.0.0",
+  "$id": "EVA_metadata_schema",
+  "author": "EVA",
+  "type": "object",
+  "title": "EVA metadata schema",
+  "description": "",
+  "required": [
+    "submitterDetails",
+    "project",
+    "sample",
+    "analysis",
+    "files"
+  ],
+  "properties": {
+    "submitterDetails": {
+      "type": "array",
+      "description": "Contact details of the submitter",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "lastName",
+          "firstName",
+          "email",
+          "laboratory",
+          "centre"
+        ],
+        "properties": {
+          "lastName": {
+            "type": "string",
+            "description": "Last name of the contact"
+          },
+          "firstName": {
+            "type": "string",
+            "description": "First name of the contact"
+          },
+          "telephone": {
+            "type": "string",
+            "description": "Contact's phone number"
+          },
+          "email": {
+            "type": "string",
+            "format": "email",
+            "description": "Contact's Email",
+            "examples": [
+              "user@domain.com"
+            ]
+          },
+          "laboratory": {
+            "description": "Name of the laboratory or department of the contact",
+            "type": "string"
+          },
+          "centre": {
+            "description": "Name of the Centre, company or institute the contact belongs to",
+            "type": "string"
+          },
+          "address": {
+            "description": "Address of the Centre, company or institute the contact belongs to",
+            "type": "string"
+          }
+        }
+      }
+    },
+    "project": {
+      "type": "object",
+      "description": "General information about the Project",
+      "required": [
+        "title",
+        "description",
+        "taxId",
+        "centre"
+      ],
+      "properties": {
+        "title": {
+          "type": "string",
+          "description": "Title of the project"
+        },
+        "description": {
+          "type": "string",
+          "description": "Description of the project"
+        },
+        "centre": {
+          "type": "string",
+          "description": "Centre, institute or company submitting the project"
+        },
+        "taxId": {
+          "type": "number",
+          "description": "Taxonomy ID of the species associated with the project (found here: http://www.ncbi.nlm.nih.gov/taxonomy)"
+        },
+        "publications": {
+          "type": "array",
+          "items": true
+        },
+        "parentProject": {
+          "$ref": "#/definitions/projectAccession"
+        },
+        "childProjects": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/projectAccession"
+          }
+        },
+        "peerProjects": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/projectAccession"
+          }
+        },
+        "links": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "holdDate": {
+          "type": "string",
+          "format": "date",
+          "examples": [
+            "2018-11-13"
+          ]
+        }
+      }
+    },
+    "analysis": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": [
+          "analysisTitle",
+          "analysisAlias",
+          "description",
+          "experimentType",
+          "referenceGenome"
+        ],
+        "properties": {
+          "analysisTitle": {
+            "type": "string",
+            "description": "Title of the analysis"
+          },
+          "analysisAlias": {
+            "type": "string",
+            "description": "Shortened identifier for the analysis"
+          },
+          "description": {
+            "type": "string",
+            "description": "Description of the analysis"
+          },
+          "experimentType": {
+            "type": "string",
+            "description": "Type of experiment",
+            "enum": [
+              "Whole genome sequencing",
+              "Whole transcriptome sequencing",
+              "Exome sequencing",
+              "Genotyping by array",
+              "Curation",
+              "Genotyping by sequencing",
+              "Target sequencing",
+              "Transcriptomics"
+            ]
+          },
+          "referenceGenome": {
+            "type": "string",
+            "description": "Reference against which the analysis was performed. ENA or NCBI accession (starting with GCA) accepted."
+          },
+           "referenceFasta": {
+            "type": "string",
+            "description": "The path to the reference in fasta format that was used to create the VCF. (This is a local path)"
+          },
+           "assemblyReport": {
+            "type": "string",
+            "description": "The path to the assembly report associated with the assembly as found in NCBI assemblies. (This is a local path and only applicable for GCAs)"
+          },
+          "platform": {
+            "description": "Platform used to generate the raw data.",
+            "type": "string",
+            "examples": [
+              "454 GS",
+              "454 GS 20",
+              "454 GS FLX",
+              "454 GS FLX Titanium",
+              "454 GS FLX+",
+              "454 GS Junior",
+              "AB 3130 Genetic Analyzer",
+              "AB 3300 Genetic Analyzer",
+              "AB 3730xl",
+              "AB 5500 Genetic Analyzer",
+              "AB 5500xl Genetic Analyzer",
+              "AB SOLiD 3 Plus System",
+              "AB SOLiD 4 System",
+              "AB SOLiD 4hq System",
+              "AB SOLiD PI System",
+              "AB SOLiD System",
+              "AB SOLiD System 2.0",
+              "AB SOLiD System 3.0",
+              "ABI 3500 Genetic Analyzer",
+              "ABI PRISM 310 Genetic Analyzer",
+              "Affymetrix",
+              "BGISEQ-500",
+              "Bio-Rad CFX96",
+              "Complete Genomics",
+              "Illumina",
+              "Illumina CanineHD",
+              "Illumina Genome Analyzer",
+              "Illumina Genome Analyzer II",
+              "Illumina Genome Analyzer IIx",
+              "Illumina HiScanSQ",
+              "Illumina HiSeq 1000",
+              "Illumina HiSeq 2000",
+              "Illumina HiSeq 2500",
+              "Illumina HiSeq 3500",
+              "Illumina HiSeq 4000",
+              "Illumina HiSeq X Ten",
+              "Illumina Hiseq Xten",
+              "Illumina iScan",
+              "Illumina MiniSeq",
+              "Illumina MiSeq",
+              "Illumina NextSeq 500",
+              "Illumina NovaSeq 6000",
+              "Ion Personal Genome Machine (PGM) System v2",
+              "Ion S5XL",
+              "Ion Torrent PGM",
+              "Ion Torrent Proton",
+              "MGISEQ-2000",
+              "Nimblegen 4.2M Probe Custom DNA Microarray",
+              "Oxford Nanopore PromethION",
+              "unspecified"
+            ]
+          },
+          "software": {
+            "description": "Software used to generate the analysis.",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "pipelineDescriptions": {
+            "type": "string"
+          },
+          "imputation": {
+            "description": "Specify if imputation was used to generate the variants",
+            "type": "boolean"
+          },
+          "phasing": {
+            "description": "Specify if the variants have been phased",
+            "type": "boolean"
+          },
+          "centre": {
+            "description": "Centre, institute or company where the analysis was performed (if different from the project's centre).",
+            "type": "string"
+          },
+          "date": {
+            "description": "Date in which the analysis was performed",
+            "type": "string"
+          },
+          "links": {
+            "description": "Link(s) to external resources related to this analysis in the form DB:ID:LABEL.",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "runAccessions": {
+            "type": "array",
+            "items": {
+              "type": "string",
+              "pattern": "^(E|D|S)RR[0-9]{6,}$"
+            },
+            "description": "List of run accessions linking to the raw data used in this analysis if applicable (e.g. SRR576651, SRR576652)"
+          }
+        }
+      }
+    },
+    "sample": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "oneOf": [
+          {
+            "required": [
+              "analysisAlias",
+              "sampleInVCF",
+              "bioSampleAccession"
+            ]
+          },
+          {
+            "required": [
+              "analysisAlias",
+              "sampleInVCF",
+              "bioSampleObject"
+            ]
+          }
+        ],
+        "properties": {
+          "analysisAlias": {
+            "type": "array",
+            "description": "Aliases of the analyses performed on this sample",
+            "minItems": 1,
+            "items": {
+              "type": "string"
+            }
+          },
+          "sampleInVCF": {
+            "type": "string",
+            "description": "Sample Name used in the VCF file"
+          },
+          "sampleAccession": {
+            "type": "string",
+            "description": "Accession of the sample (BioSamples, ENA or EGA)"
+          },
+          "bioSampleObject": {
+            "type": "object",
+            "anyOf": [
+              {
+                "$ref": "eva-biosamples.json"
+              },
+              {
+                 "$ref": "https://www.ebi.ac.uk/biosamples/schema/store/registry/schemas/ERC000011"
+              },
+              {
+                "$ref": "https://www.ebi.ac.uk/biosamples/schemas/certification/plant-miappe.json"
+              }
+            ]
+          }
+        }
+      }
+    },
+    "files": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": [
+          "analysisAlias",
+          "fileName"
+        ],
+        "properties": {
+          "analysisAlias": {
+            "type": "string",
+            "description": "Alias of the analysis that produced the file"
+          },
+          "fileName": {
+            "type": "string",
+            "description": "File name"
+          },
+          "fileSize": {
+            "type": "number",
+            "description": "Size of file in bytes"
+          },
+          "md5": {
+            "type": "string",
+            "description": "MD5 hash of the file"
+          }
+        }
+      }
+    }
+  },
+  "definitions": {
+    "projectAccession": {
+      "type": "string",
+      "description": "ENA project accession",
+      "pattern": "^PRJ(EB|NA)\\d+$"
+    }
+  }
+}

eva_sub_cli/etc/spreadsheet2json_conf.yaml

@@ -0,0 +1,135 @@
+#Configuration for defining which fields in which worksheets to be parsed in an Excel file
+#
+#At the top level, "worksheets" defines a list of worksheets (titles) to be parsed.
+#Then for each worksheet, you define which fields to be parsed from this worksheet.
+#These fields are either "required", "optional" and/or header_row. "required" fields must be found
+#in the worksheet so that this worksheet is valid. Otherwise this worksheet is not
+#valid. The field names must match those values in the header row in the worksheet.
+#
+#This is a YAML file
+
+# List of worksheets to be parsed.
+worksheets:
+  Submitter Details: submitterDetails
+  Project: project
+  Analysis: analysis
+  Sample: sample
+  Files: files
+
+Submitter Details:
+  required:
+    Last Name: lastName
+    First Name: firstName
+    Email Address: email
+    Laboratory: laboratory
+    Center: centre
+  optional:
+    Address: address
+
+# List of field for Projects
+Project:
+  required:
+    Project Title: title
+    Description: description
+    Center: centre
+    Tax ID: taxId
+  optional:
+    Publication(s): publications
+    Parent Project: parentProject
+    Child Project(s): childProjects
+    Peer Project(s): peerProjects
+    Link(s): links
+    Hold Date: holdDate
+    Collaborator(s): collaborators
+    Strain: strain
+    Breed: breed
+    Broker: broker
+  cast:
+    Publication(s): list
+    Child Project(s): list
+    Peer Project(s): list
+    Link(s): list
+    Hold Date: date
+
+Analysis:
+  required:
+    Analysis Title: analysisTitle
+    Analysis Alias: analysisAlias
+    Description: description
+    Experiment Type: experimentType
+    Reference: referenceGenome
+  optional:
+    Reference Fasta Path: referenceFasta
+    Platform: platform
+    Software: software
+    Pipeline Description: pipelineDescriptions
+    Imputation: imputation
+    Phasing: phasing
+    Centre: centre
+    Date: date
+    Link(s): links
+    Run Accession(s): runAccessions
+  cast:
+    Run Accession(s): list
+    Imputation: boolean
+    Phasing: boolean
+    Date: date
+
+Sample:
+  header_row: 3
+  required:
+    Analysis Alias: analysisAlias
+    Sample Name in VCF: sampleInVCF
+  optional:
+    Sample Accession: bioSampleAccession
+    BioSample Name: bioSampleName
+    Title: title
+    Description: description
+    Unique Name: uniqueName
+    Prefix: prefix
+    Subject: subject
+    Derived From: derivedFrom
+    Tax Id: taxId
+    Scientific Name: scientificName
+    Common Name: commonName
+    mating_type: matingType
+    sex: sex
+    population: population
+    cell_type: cellType
+    dev_stage: devStage
+    germline: germline
+    tissue_lib: tissueLib
+    tissue_type: tissueType
+    bio_material: BioMaterial
+    culture_collection: cultureCollection
+    specimen_voucher: specimenVoucher
+    collected_by: collectedBy
+    collection_date: collectionDate
+    geographic location (country and/or sea): geographicLocationCountrySea
+    geographic location (region and locality): geographicLocationRegion
+    host: host
+    identified_by: identifiedBy
+    isolation_source: isolationSource
+    lat_lon: latLon
+    lab_host: LabHost
+    environmental_sample: environmentalSample
+    cultivar: cultivar
+    ecotype: ecotype
+    isolate: isolate
+    strain: strain
+    sub_species: subSpecies
+    variety: variety
+    sub_strain: subStrain
+    cell_line: cellType
+    serotype: serotype
+    serovar: serovar
+  cast:
+    Sample ID: string
+    BioSample Name: string
+    collection_date: date
+
+
+Files:
+  required:
+    Analysis Alias: analysisAlias
+    File Name: fileName

eva_sub_cli/exceptions/invalid_file_type_exception.py

@@ -0,0 +1,4 @@
+class InvalidFileTypeError(Exception):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)

eva_sub_cli/exceptions/submission_not_found_exception.py

@@ -0,0 +1,4 @@
+class SubmissionNotFoundException(Exception):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)

eva_sub_cli/exceptions/submission_status_exception.py

@@ -0,0 +1,5 @@
+
+class SubmissionStatusException(Exception):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)

eva_sub_cli/executables/check_fasta_insdc.py

@@ -0,0 +1,159 @@
+import argparse
+import gzip
+import hashlib
+import json
+from itertools import groupby
+
+import requests
+import yaml
+from ebi_eva_common_pyutils.logger import logging_config
+from requests import HTTPError
+from retry import retry
+
+from eva_sub_cli.file_utils import fasta_iter
+from eva_sub_cli.metadata_utils import get_files_per_analysis, get_analysis_for_vcf_file, \
+    get_reference_assembly_for_analysis
+
+REFGET_SERVER = 'https://www.ebi.ac.uk/ena/cram'
+CONTIG_ALIAS_SERVER = 'https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum'
+
+logger = logging_config.get_logger(__name__)
+
+
+def write_result_yaml(output_yaml, results):
+    with open(output_yaml, 'w') as open_yaml:
+        yaml.safe_dump(data=results, stream=open_yaml)
+
+
+def refget_md5_digest(sequence):
+    return hashlib.md5(sequence.upper().encode('utf-8')).hexdigest()
+
+@retry(exceptions=(HTTPError,), tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+def get_refget_metadata(md5_digest):
+    response = requests.get(f'{REFGET_SERVER}/sequence/{md5_digest}/metadata')
+    if 500 <= response.status_code < 600:
+        raise HTTPError(f"{response.status_code} Server Error: {response.reason} for url: {response.url}", response=response)
+    if 200 <= response.status_code < 300:
+        return response.json()
+    return None
+
+
+@retry(exceptions=(HTTPError,), tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+def _get_containing_assemblies_paged(url):
+    response = requests.get(url)
+    if 500 <= response.status_code < 600:
+        raise HTTPError(f"{response.status_code} Server Error: {response.reason} for url: {response.url}",
+                        response=response)
+    if 200 <= response.status_code < 300:
+        results = set()
+        response_data = response.json()
+        if '_embedded' in response_data and 'chromosomeEntities' in response_data['_embedded']:
+            for contigEntity in response_data['_embedded']['chromosomeEntities']:
+                results.add(contigEntity['assembly']['insdcAccession'])
+        if '_links' in response_data and 'next' in response_data['_links']:
+            # Add results from next page if needed
+            results |= _get_containing_assemblies_paged(response_data['_links']['next'])
+        return results
+    return set()
+
+
+def get_containing_assemblies(md5_digest):
+    # Wrapper method to handle pagination
+    url = f'{CONTIG_ALIAS_SERVER}/{md5_digest}'
+    return _get_containing_assemblies_paged(url)
+
+
+def assess_fasta(input_fasta, analyses, assembly_in_metadata):
+    """
+    Check whether all sequences in fasta file are INSDC, and if so whether the INSDC accession provided in the metadata
+    is compatible.
+    :param input_fasta: path to fasta file
+    :param analyses: aliases of all analyses associated with this fasta (used only for reporting)
+    :param assembly_in_metadata: INSDC accession from metadata (if None will only do the first check but will report
+      compatible assemblies)
+    :returns: dict of results
+    """
+    results = {'sequences': []}
+    all_insdc = True
+    possible_assemblies = set()
+    try:
+        for header, sequence in fasta_iter(input_fasta):
+            # Check sequence is INSDC
+            name = header.split()[0]
+            md5_digest = refget_md5_digest(sequence)
+            sequence_metadata = get_refget_metadata(md5_digest)
+            is_insdc = bool(sequence_metadata)
+            results['sequences'].append({'sequence_name': name, 'sequence_md5': md5_digest, 'insdc': is_insdc})
+            all_insdc = all_insdc and is_insdc
+            # Get possible assemblies for sequence
+            containing_assemblies = get_containing_assemblies(md5_digest)
+            if len(containing_assemblies) == 0:
+                if is_insdc:
+                    logger.warning(f'Sequence with this MD5 is INSDC but not found in contig alias: {md5_digest}')
+            elif len(possible_assemblies) == 0:
+                possible_assemblies = containing_assemblies
+            else:
+                possible_assemblies &= containing_assemblies
+    except (ConnectionError, HTTPError) as e:
+        # Server errors from either ENA refget or EVA contig alias will halt the check prematurely.
+        # Report the error but do not return from the method, so that incomplete results can be reported
+        # (i.e. any sequences found to be INSDC and any compatible assemblies so far)
+        results['connection_error'] = str(e)
+
+    # Always report whether everything is INSDC
+    results['all_insdc'] = all_insdc
+    # Only report compatible assembly accessions if any were found in contig alias
+    if possible_assemblies:
+        results['possible_assemblies'] = possible_assemblies
+    # Only report on metadata concordance if we found compatible assemblies and found a single assembly accession in
+    # the metadata to compare against this FASTA file
+    if possible_assemblies and assembly_in_metadata:
+        results['metadata_assembly_compatible'] = (assembly_in_metadata in possible_assemblies)
+        results['associated_analyses'] = analyses
+        results['assembly_in_metadata'] = assembly_in_metadata
+    return results
+
+
+def get_analyses_and_reference_genome_from_metadata(vcf_files_for_fasta, json_file):
+    """
+    Get analysis aliases and associated reference genome from the metadata.
+    :param vcf_files_for_fasta: list of VCF file paths, assumed to be associated with a single FASTA file and thus a
+      single assembly accession
+    :param json_file: JSON file of the metadata
+    """
+    with open(json_file) as open_json:
+        metadata = json.load(open_json)
+        files_per_analysis = get_files_per_analysis(metadata)
+        # Get all analyses associated with all vcf files that are linked with a single fasta file
+        all_analyses = set()
+        for vcf_file in vcf_files_for_fasta:
+            analysis_aliases = get_analysis_for_vcf_file(vcf_file, files_per_analysis)
+            if len(analysis_aliases) != 1:
+                logger.error(f'Could not determine analysis associated with VCF file: {vcf_file}')
+            else:
+                all_analyses.add(analysis_aliases[0])
+        # Get (single) assembly associated with these analyses
+        assemblies = [get_reference_assembly_for_analysis(metadata, analysis) for analysis in all_analyses]
+        if len(assemblies) != 1:
+            logger.error(f'Could not determine assembly accession to check against fasta file, out of: {assemblies}')
+            return all_analyses, None
+        return all_analyses, assemblies[0]
+
+
+def main():
+    arg_parser = argparse.ArgumentParser(
+        description="Calculate each sequence's Refget MD5 digest and compare these against INSDC Refget server.")
+    arg_parser.add_argument('--input_fasta', required=True, dest='input_fasta',
+                            help='Fasta file that contains the sequence to be checked')
+    arg_parser.add_argument('--vcf_files', dest='vcf_files', nargs='+',
+                            help='VCF files associated with this fasta file (used to connect fasta file to '
+                                 'assembly accession in metadata via analysis)')
+    arg_parser.add_argument('--metadata_json', required=True, dest='metadata_json', help='EVA metadata json file')
+    arg_parser.add_argument('--output_yaml', required=True, dest='output_yaml',
+                            help='Path to the location of the results')
+    args = arg_parser.parse_args()
+    logging_config.add_stdout_handler()
+    analyses, metadata_insdc = get_analyses_and_reference_genome_from_metadata(args.vcf_files, args.metadata_json)
+    results = assess_fasta(args.input_fasta, analyses, metadata_insdc)
+    write_result_yaml(args.output_yaml, results)
+