eva-sub-cli 0.4.4__py3-none-any.whl

eva_sub_cli/orchestrator.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python
+import csv
+import json
+import os
+from collections import defaultdict
+
+import requests
+from ebi_eva_common_pyutils.config import WritableConfig
+from ebi_eva_common_pyutils.logger import logging_config
+from openpyxl.reader.excel import load_workbook
+
+from eva_sub_cli import SUB_CLI_CONFIG_FILE, __version__
+from eva_sub_cli.exceptions.invalid_file_type_exception import InvalidFileTypeError
+from eva_sub_cli.exceptions.submission_not_found_exception import SubmissionNotFoundException
+from eva_sub_cli.exceptions.submission_status_exception import SubmissionStatusException
+from eva_sub_cli.submission_ws import SubmissionWSClient
+from eva_sub_cli.submit import StudySubmitter, SUB_CLI_CONFIG_KEY_SUBMISSION_ID, \
+    SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL
+from eva_sub_cli.validators.docker_validator import DockerValidator
+from eva_sub_cli.validators.native_validator import NativeValidator
+from eva_sub_cli.validators.validator import READY_FOR_SUBMISSION_TO_EVA
+
+VALIDATE = 'validate'
+SUBMIT = 'submit'
+DOCKER = 'docker'
+NATIVE = 'native'
+
+logger = logging_config.get_logger(__name__)
+
+
+def get_vcf_files(mapping_file):
+    vcf_files = []
+    with open(mapping_file) as open_file:
+        reader = csv.DictReader(open_file, delimiter=',')
+        for row in reader:
+            vcf_files.append(row['vcf'])
+    return vcf_files
+
+
+def get_project_title_and_create_vcf_files_mapping(submission_dir, vcf_files, reference_fasta,
+                                                   metadata_json, metadata_xlsx):
+    """
+    Get project title and mapping between VCF files and reference FASTA files, from three sources: command line
+        arguments, metadata JSON file, or metadata XLSX file.
+
+    :param submission_dir: Directory where mapping file will be saved
+    :param vcf_files: VCF files from command line, if present
+    :param reference_fasta: Reference FASTA from command line, if present
+    :param metadata_json: Metadata JSON from command line, if present
+    :param metadata_xlsx: Metadata XLSX from command line, if present
+    :return: Project title and path to the mapping file
+    """
+    mapping_file = os.path.join(submission_dir, 'vcf_mapping_file.csv')
+    with open(mapping_file, 'w') as open_file:
+        writer = csv.writer(open_file, delimiter=',')
+        writer.writerow(['vcf', 'fasta', 'report'])
+
+        vcf_files_mapping = []
+        if vcf_files and reference_fasta:
+            for vcf_file in vcf_files:
+                vcf_files_mapping.append([os.path.abspath(vcf_file), os.path.abspath(reference_fasta), ''])
+            if metadata_json:
+                project_title, _ = get_project_and_vcf_fasta_mapping_from_metadata_json(metadata_json, False)
+            elif metadata_xlsx:
+                project_title, _ = get_project_and_vcf_fasta_mapping_from_metadata_xlsx(metadata_xlsx, False)
+        elif metadata_json:
+            project_title, vcf_files_mapping = get_project_and_vcf_fasta_mapping_from_metadata_json(metadata_json, True)
+        elif metadata_xlsx:
+            project_title, vcf_files_mapping = get_project_and_vcf_fasta_mapping_from_metadata_xlsx(metadata_xlsx, True)
+
+        validate_vcf_mapping(vcf_files_mapping)
+        for mapping in vcf_files_mapping:
+            writer.writerow(mapping)
+
+    return project_title, mapping_file
+
+
+def validate_vcf_mapping(vcf_mapping):
+    """
+    Validate that VCF files and FASTA files in the mapping are present and FASTA files are not compressed.
+
+    :param vcf_mapping: iterable of triples (VCF file path, reference FASTA path, optional assembly report path)
+    :return:
+    """
+    for vcf_file, fasta_file, report_file in vcf_mapping:
+        if not (vcf_file and os.path.isfile(vcf_file)):
+            raise FileNotFoundError(f'The variant file {vcf_file} does not exist, please check the file path.')
+        if not (fasta_file and os.path.isfile(fasta_file)):
+            raise FileNotFoundError(f'The reference fasta {fasta_file} does not exist, please check the file path.')
+        if fasta_file.lower().endswith('gz'):
+            raise InvalidFileTypeError(f'The reference fasta {fasta_file} is compressed, please uncompress the file.')
+        if report_file and not os.path.isfile(report_file):
+            raise FileNotFoundError(f'The assembly report file {report_file} does not exist, please check the file '
+                                    f'path.')
+
+
+def get_project_and_vcf_fasta_mapping_from_metadata_json(metadata_json, mapping_req=False):
+    with open(metadata_json) as file:
+        json_metadata = json.load(file)
+
+        if 'project' in json_metadata:
+            if 'title' in json_metadata['project']:
+                project_title = json_metadata['project']['title']
+
+        vcf_fasta_report_mapping = []
+        if mapping_req:
+            analysis_alias_dict = defaultdict(dict)
+            for analysis in json_metadata['analysis']:
+                analysis_alias_dict[analysis['analysisAlias']]['referenceFasta'] = analysis['referenceFasta']
+                analysis_alias_dict[analysis['analysisAlias']]['assemblyReport'] = analysis['assemblyReport'] \
+                    if 'assemblyReport' in analysis else ''
+
+            for file_dict in json_metadata['files']:
+                reference_fasta = analysis_alias_dict[file_dict['analysisAlias']]['referenceFasta']
+                assembly_report = analysis_alias_dict[file_dict['analysisAlias']]['assemblyReport']
+                vcf_fasta_report_mapping.append([os.path.abspath(file_dict['fileName']),
+                                                 os.path.abspath(reference_fasta),
+                                                 os.path.abspath(assembly_report) if assembly_report else ''])
+
+    return project_title, vcf_fasta_report_mapping
+
+
+def get_project_and_vcf_fasta_mapping_from_metadata_xlsx(metadata_xlsx, mapping_req=False):
+    workbook = load_workbook(metadata_xlsx)
+
+    project_sheet = workbook['Project']
+    project_headers = {}
+    for cell in project_sheet[1]:
+        project_headers[cell.value] = cell.column
+    project_title = project_sheet.cell(row=2, column=project_headers['Project Title']).value
+
+    vcf_fasta_report_mapping = []
+    if mapping_req:
+        analysis_alias_sheet = workbook['Analysis']
+        analysis_headers = {}
+        for cell in analysis_alias_sheet[1]:
+            analysis_headers[cell.value] = cell.column - 1
+
+        analysis_alias_dict = {}
+        for row in analysis_alias_sheet.iter_rows(min_row=2, values_only=True):
+            analysis_alias = row[analysis_headers['Analysis Alias']]
+            reference_fasta = row[analysis_headers['Reference Fasta Path']]
+            analysis_alias_dict[analysis_alias] = reference_fasta
+
+        files_sheet = workbook['Files']
+        files_headers = {}
+        for cell in files_sheet[1]:
+            files_headers[cell.value] = cell.column - 1
+
+        for row in files_sheet.iter_rows(min_row=2, values_only=True):
+            file_name = os.path.abspath(row[files_headers['File Name']])
+            analysis_alias = row[files_headers['Analysis Alias']]
+            reference_fasta = os.path.abspath(analysis_alias_dict[analysis_alias])
+            vcf_fasta_report_mapping.append([os.path.abspath(file_name), os.path.abspath(reference_fasta), ''])
+
+    return project_title, vcf_fasta_report_mapping
+
+
+def check_validation_required(tasks, sub_config, username=None, password=None):
+    # Validation is mandatory so if submit is requested then VALIDATE must have run before or be requested as well
+    if SUBMIT in tasks:
+        if not sub_config.get(READY_FOR_SUBMISSION_TO_EVA, False):
+            return True
+        # If we are working with an existing submission check its status to see if it was submitted and failed before.
+        submission_id = sub_config.get(SUB_CLI_CONFIG_KEY_SUBMISSION_ID, None)
+        if submission_id:
+            try:
+                submission_status = SubmissionWSClient(username, password).get_submission_status(submission_id)
+                if submission_status == 'FAILED':
+                    # Reset the submission_id which will force the creation of a new one
+                    sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_ID, value=None)
+                    sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL, value=None)
+                    return True
+                else:
+                    return False
+            except requests.HTTPError as ex:
+                if ex.response.status_code == 404:
+                    logger.error(
+                        f'Submission with id {submission_id} could not be found: '
+                        f'status code: {ex.response.status_code} response: {ex.response.text}')
+                    raise SubmissionNotFoundException(f'Submission with id {submission_id} could not be found')
+                else:
+                    logger.error(f'Error occurred while getting status of the submission with Id {submission_id}: '
+                                 f'status code: {ex.response.status_code} response: {ex.response.text}')
+                    raise SubmissionStatusException(f'Error occurred while getting status of the submission '
+                                                    f'with Id {submission_id}')
+
+        logger.debug(f'submission id not found in config. This might be the first time user is submitting')
+        return False
+
+
+def orchestrate_process(submission_dir, vcf_files, reference_fasta, metadata_json, metadata_xlsx,
+                        tasks, executor, username=None, password=None, shallow_validation=False, **kwargs):
+    # load config
+    config_file_path = os.path.join(submission_dir, SUB_CLI_CONFIG_FILE)
+    sub_config = WritableConfig(config_file_path, version=__version__)
+
+    metadata_file = metadata_json or metadata_xlsx
+    if not os.path.exists(os.path.abspath(metadata_file)):
+        raise FileNotFoundError(f'The provided metadata file {os.path.abspath(metadata_file)} does not exist')
+
+    if metadata_json:
+        metadata_json = os.path.abspath(metadata_json)
+    if metadata_xlsx:
+        metadata_xlsx = os.path.abspath(metadata_xlsx)
+
+    # Get the provided Project Title and VCF files mapping (VCF, Fasta and Report)
+    project_title, vcf_files_mapping = get_project_title_and_create_vcf_files_mapping(
+        submission_dir, vcf_files, reference_fasta, metadata_json, metadata_xlsx
+    )
+    vcf_files = get_vcf_files(vcf_files_mapping)
+
+    if VALIDATE not in tasks and check_validation_required(tasks, sub_config, username, password):
+        tasks.append(VALIDATE)
+
+    if VALIDATE in tasks:
+        if executor == DOCKER:
+            validator = DockerValidator(vcf_files_mapping, submission_dir, project_title, metadata_json, metadata_xlsx,
+                                        shallow_validation=shallow_validation, submission_config=sub_config)
+        # default to native execution
+        else:
+            validator = NativeValidator(vcf_files_mapping, submission_dir, project_title, metadata_json, metadata_xlsx,
+                                        shallow_validation=shallow_validation, submission_config=sub_config)
+        with validator:
+            validator.validate_and_report()
+            if not metadata_json:
+                metadata_json = os.path.join(validator.output_dir, 'metadata.json')
+            sub_config.set('metadata_json', value=metadata_json)
+            sub_config.set('vcf_files', value=vcf_files)
+
+    if SUBMIT in tasks:
+        with StudySubmitter(submission_dir, submission_config=sub_config, username=username, password=password) as submitter:
+            submitter.submit()

eva_sub_cli/report.py

@@ -0,0 +1,40 @@
+import base64
+import os.path
+import re
+
+from jinja2 import Environment, FileSystemLoader
+
+import eva_sub_cli
+
+current_dir = os.path.dirname(__file__)
+
+
+def get_logo_data():
+    with open(os.path.join(current_dir, "etc/eva_logo.png"), "rb") as f:
+        logo_data = base64.b64encode(f.read()).decode("utf-8")
+        return logo_data
+
+
+def generate_html_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping,
+                         project_title=None):
+    vcf_files = sorted(set([file_name
+                            for check in validation_results if check in ["vcf_check", "assembly_check"]
+                            for file_name in validation_results[check]
+                            ]))
+    fasta_files = sorted([file_name for file_name in validation_results['fasta_check']])
+    template = Environment(
+        loader=FileSystemLoader(os.path.join(current_dir, 'jinja_templates'))
+    ).get_template('html_report.html')
+    rendered_template = template.render(
+        cli_version=eva_sub_cli.__version__,
+        logo_data=get_logo_data(),
+        project_title=project_title,
+        validation_date=validation_date,
+        vcf_files=vcf_files,
+        fasta_files=fasta_files,
+        submission_dir=submission_dir,
+        vcf_fasta_analysis_mapping=vcf_fasta_analysis_mapping,
+        validation_results=validation_results
+    )
+    return re.sub('\s+\n', '\n', rendered_template)
+

eva_sub_cli/semantic_metadata.py

@@ -0,0 +1,243 @@
+import json
+from collections import Counter
+from copy import deepcopy
+
+import yaml
+
+from retry import retry
+from ebi_eva_common_pyutils.biosamples_communicators import NoAuthHALCommunicator
+from ebi_eva_common_pyutils.ena_utils import download_xml_from_ena, get_scientific_name_and_common_name
+from ebi_eva_common_pyutils.logger import AppLogger
+
+from eva_sub_cli.date_utils import check_date
+
+PROJECT_KEY = 'project'
+ANALYSIS_KEY = 'analysis'
+SAMPLE_KEY = 'sample'
+FILES_KEY = 'files'
+PARENT_PROJECT_KEY = 'parentProject'
+CHILD_PROJECTS_KEY = 'childProjects'
+PEER_PROJECTS_KEY = 'peerProjects'
+BIOSAMPLE_OBJECT_KEY = 'bioSampleObject'
+BIOSAMPLE_ACCESSION_KEY = 'bioSampleAccession'
+CHARACTERISTICS_KEY = 'characteristics'
+TAX_ID_KEY = 'taxId'
+SCI_NAME_KEYS = ['species', 'Species', 'organism', 'Organism']
+ANALYSIS_ALIAS_KEY = 'analysisAlias'
+ANALYSIS_RUNS_KEY = 'runAccessions'
+
+
+def cast_list(l, type_to_cast=str):
+    for e in l:
+        yield type_to_cast(e)
+
+
+class SemanticMetadataChecker(AppLogger):
+
+    def __init__(self, metadata, sample_checklist='ERC000011'):
+        self.sample_checklist = sample_checklist
+        self.metadata = metadata
+        self.errors = []
+        # Caches whether taxonomy code is valid or not, and maps to scientific name if valid
+        self.taxonomy_valid = {}
+        self.communicator = NoAuthHALCommunicator(bsd_url='https://www.ebi.ac.uk/biosamples')
+
+    def write_result_yaml(self, output_path):
+        with open(output_path, 'w') as open_yaml:
+            yaml.safe_dump(data=self.errors, stream=open_yaml)
+
+    def check_all(self):
+        self.check_all_project_accessions()
+        self.check_all_taxonomy_codes()
+        self.check_all_scientific_names()
+        self.check_existing_biosamples()
+        self.check_all_analysis_run_accessions()
+        self.check_analysis_alias_coherence()
+
+    def check_all_project_accessions(self):
+        """Check that ENA project accessions exist and are public."""
+        project = self.metadata[PROJECT_KEY]
+        if PARENT_PROJECT_KEY in project:
+            self.check_project_accession(project[PARENT_PROJECT_KEY], f'/{PROJECT_KEY}/{PARENT_PROJECT_KEY}')
+        for idx, accession in enumerate(project.get(CHILD_PROJECTS_KEY, [])):
+            self.check_project_accession(accession, f'/{PROJECT_KEY}/{CHILD_PROJECTS_KEY}/{idx}')
+        for idx, accession in enumerate(project.get(PEER_PROJECTS_KEY, [])):
+            self.check_project_accession(accession, f'/{PROJECT_KEY}/{PEER_PROJECTS_KEY}/{idx}')
+
+    def check_all_taxonomy_codes(self):
+        """Check that taxonomy IDs are valid according to ENA."""
+        project = self.metadata[PROJECT_KEY]
+        if TAX_ID_KEY in project:
+            self.check_taxonomy_code(project[TAX_ID_KEY], f'/{PROJECT_KEY}/{TAX_ID_KEY}')
+        # Check sample taxonomies for novel samples
+        for idx, sample in enumerate(self.metadata[SAMPLE_KEY]):
+            if BIOSAMPLE_OBJECT_KEY in sample and TAX_ID_KEY in sample[BIOSAMPLE_OBJECT_KEY][CHARACTERISTICS_KEY]:
+                self.check_taxonomy_code(sample[BIOSAMPLE_OBJECT_KEY][CHARACTERISTICS_KEY][TAX_ID_KEY][0]['text'],
+                                         f'/{SAMPLE_KEY}/{idx}/{BIOSAMPLE_OBJECT_KEY}/{CHARACTERISTICS_KEY}/{TAX_ID_KEY}')
+
+    def check_uniqueness_analysis_alias(self):
+        analyses = self.metadata[ANALYSIS_KEY]
+        analysis_aliases = [analysis.get(ANALYSIS_ALIAS_KEY) for analysis in analyses]
+        if len(analysis_aliases) != len(set(analysis_aliases)):
+            counter = Counter(analysis_aliases)
+            for idx, analysis in enumerate(analyses):
+                if counter[analysis.get(ANALYSIS_ALIAS_KEY)] > 1:
+                    self.add_error(
+                        f'/{ANALYSIS_KEY}/{idx}/{ANALYSIS_ALIAS_KEY}',
+                        f'Analysis alias {analysis.get(ANALYSIS_ALIAS_KEY)} is present '
+                        f'{counter[analysis.get(ANALYSIS_ALIAS_KEY)]} times in the Analysis Sheet')
+
+    def check_all_scientific_names(self):
+        """Check that all scientific names are consistent with taxonomy codes."""
+        for idx, sample in enumerate(self.metadata[SAMPLE_KEY]):
+            if BIOSAMPLE_OBJECT_KEY in sample and TAX_ID_KEY in sample[BIOSAMPLE_OBJECT_KEY][CHARACTERISTICS_KEY]:
+                characteristics = sample[BIOSAMPLE_OBJECT_KEY][CHARACTERISTICS_KEY]
+                # Get the scientific name from the taxonomy (if valid)
+                tax_code = int(characteristics[TAX_ID_KEY][0]['text'])
+                sci_name_from_tax = self.taxonomy_valid[tax_code]
+                if not sci_name_from_tax:
+                    continue
+                # Check if scientific name in sample matches
+                for sci_name_key in SCI_NAME_KEYS:
+                    if sci_name_key in characteristics:
+                        sci_name = characteristics[sci_name_key][0]['text']
+                        if sci_name_from_tax.lower() != sci_name.lower():
+                            self.add_error(
+                                f'/{SAMPLE_KEY}/{idx}/{BIOSAMPLE_OBJECT_KEY}/{CHARACTERISTICS_KEY}/{sci_name_key}',
+                                f'Species {sci_name} does not match taxonomy {tax_code} ({sci_name_from_tax})')
+
+    def check_all_analysis_run_accessions(self):
+        """Check that the Run accession are valid and exist in ENA"""
+        for idx, analysis in enumerate(self.metadata[ANALYSIS_KEY]):
+            json_path = f'/{ANALYSIS_KEY}/{idx}/{ANALYSIS_RUNS_KEY}'
+            if ANALYSIS_RUNS_KEY in analysis and analysis[ANALYSIS_RUNS_KEY]:
+                for run_acc in analysis[ANALYSIS_RUNS_KEY]:
+                    self.check_accession_in_ena(run_acc, 'Run', json_path)
+
+    @retry(tries=4, delay=2, backoff=1.2, jitter=(1, 3))
+    def check_accession_in_ena(self, ena_accession, accession_type, json_path):
+        try:
+            res = download_xml_from_ena(f'https://www.ebi.ac.uk/ena/browser/api/xml/{ena_accession}')
+        except Exception:
+            self.add_error(json_path, f'{accession_type} {ena_accession} does not exist in ENA or is private')
+
+    def check_project_accession(self, project_acc, json_path):
+        self.check_accession_in_ena(project_acc, 'Project', json_path)
+
+    @retry(tries=4, delay=2, backoff=1.2, jitter=(1, 3))
+    def check_taxonomy_code(self, taxonomy_code, json_path):
+        taxonomy_code = int(taxonomy_code)
+        if taxonomy_code in self.taxonomy_valid:
+            if self.taxonomy_valid[taxonomy_code] is False:
+                self.add_error(json_path, f'{taxonomy_code} is not a valid taxonomy code')
+        else:
+            try:
+                sci_name, _ = get_scientific_name_and_common_name(taxonomy_code)
+                self.taxonomy_valid[taxonomy_code] = sci_name
+            except Exception:
+                self.add_error(json_path, f'{taxonomy_code} is not a valid taxonomy code')
+                self.taxonomy_valid[taxonomy_code] = False
+
+    def add_error(self, property, description):
+        """
+        Add an error, conforming to the format of biovalidator errors.
+
+        :param property: JSON property of the error. This will be converted to sheet/row/column in spreadsheet if needed.
+        :param description: description of the error.
+        """
+        self.errors.append({'property': property, 'description': description})
+
+    def _get_biosample(self, sample_accession):
+        return self.communicator.follows_link('samples', join_url=sample_accession)
+
+    def check_existing_biosamples(self):
+        """Check that existing BioSamples are accessible and contain the required attributes."""
+        for idx, sample in enumerate(self.metadata[SAMPLE_KEY]):
+            if BIOSAMPLE_ACCESSION_KEY in sample:
+                sample_accession = sample[BIOSAMPLE_ACCESSION_KEY]
+                json_path = f'/{SAMPLE_KEY}/{idx}/{BIOSAMPLE_ACCESSION_KEY}'
+                try:
+                    sample_data = self._get_biosample(sample_accession)
+                    if self.sample_checklist:
+                        self._validate_biosample_against_checklist(sample_data, json_path, sample_accession)
+                    else:
+                        self._local_validate_existing_biosample(sample_data, json_path, sample_accession)
+
+                except ValueError:
+                    self.add_error(json_path, f'{sample_accession} does not exist or is private')
+
+    def _validate_biosample_against_checklist(self, sample_data, json_path, accession=None):
+        pre_acceptable_code = self.communicator.acceptable_code
+        try:
+            # Allow the communicator to accept failure
+            self.communicator.acceptable_code = [200, 400]
+            sample_data = deepcopy(sample_data)
+            # Add checklist to the sample
+            if 'characteristics' not in sample_data:
+                sample_data['characteristics'] = {}
+            if 'checklist' not in sample_data['characteristics']:
+                sample_data['characteristics']['checklist'] = []
+            sample_data['characteristics']['checklist'].append({'text': self.sample_checklist})
+            text = self.communicator.follows_link('samples', join_url='validate', method='POST', json=sample_data,
+                                                  text_only=True)
+            response = json.loads(text)
+            # Successful response returns the sample object, error response has a list of error objects.
+            if isinstance(response, list):
+                for error_dict in response:
+                    if accession:
+                        self.add_error(json_path, f'Existing sample {accession} {error_dict["errors"][0]}')
+                    else:
+                        self.add_error(json_path, error_dict["errors"][0])
+        finally:
+            self.communicator.acceptable_code = pre_acceptable_code
+
+    def _local_validate_existing_biosample(self, sample_data, json_path, accession):
+        """Check if the existing sample has the expected fields present"""
+        found_collection_date = False
+        for key in ['collection_date', 'collection date']:
+            if key in sample_data['characteristics'] and check_date(sample_data['characteristics'][key][0]['text']):
+                found_collection_date = True
+        if not found_collection_date:
+            self.add_error(json_path, f'Existing sample {accession} does not have a valid collection date')
+
+        found_geo_loc = False
+        for key in ['geographic location (country and/or sea)', 'geo loc name']:
+            if key in sample_data['characteristics'] and sample_data['characteristics'][key][0]['text']:
+                found_geo_loc = True
+        if not found_geo_loc:
+            self.add_error(json_path, f'Existing sample {accession} does not have a valid geographic location')
+
+    def check_analysis_alias_coherence(self):
+        """Check that the same analysis aliases are used in analysis, sample, and files."""
+        analysis_aliases = [analysis[ANALYSIS_ALIAS_KEY] for analysis in self.metadata[ANALYSIS_KEY]]
+        aliases_in_samples = [alias for sample in self.metadata[SAMPLE_KEY] for alias in sample[ANALYSIS_ALIAS_KEY]]
+        aliases_in_files = [file[ANALYSIS_ALIAS_KEY] for file in self.metadata[FILES_KEY]]
+
+        self.same_set(analysis_aliases, aliases_in_samples, 'Analysis', 'Samples',
+                      f'/{SAMPLE_KEY}/{ANALYSIS_ALIAS_KEY}')
+        self.same_set(analysis_aliases, aliases_in_files, 'Analysis', 'Files',
+                      f'/{FILES_KEY}/{ANALYSIS_ALIAS_KEY}')
+
+    def same_set(self, list1, list2, list1_desc, list2_desc, json_path):
+        """
+        Compare contents of two lists and add error messages.
+
+        :param list1: first list to compare
+        :param list2: second list to compare
+        :param list1_desc: text description of first list, used only in error message
+        :param list2_desc: text description of second list, used only in error message
+        :param json_path: property where the error message will appear
+        """
+        if not set(list1) == set(list2):
+            list1_list2 = sorted(cast_list(set(list1).difference(list2)))
+            list2_list1 = sorted(cast_list(set(list2).difference(list1)))
+            if list1_list2:
+                self.add_error(
+                    property=json_path,
+                    description=f'{",".join(list1_list2)} present in {list1_desc} not in {list2_desc}'
+                )
+            if list2_list1:
+                self.add_error(
+                    property=json_path,
+                    description=f'{",".join(list2_list1)} present in {list2_desc} not in {list1_desc}'
+                )

eva_sub_cli/submission_ws.py

@@ -0,0 +1,61 @@
+import os
+
+import requests
+from ebi_eva_common_pyutils.logger import AppLogger
+from requests import HTTPError
+from retry import retry
+
+from eva_sub_cli import SUBMISSION_WS_VAR
+from eva_sub_cli.auth import get_auth
+
+
+class SubmissionWSClient(AppLogger):
+    """
+    Python client for interfacing with the Submission WS API.
+    """
+
+    def __init__(self, username=None, password=None):
+        self.auth = get_auth(username, password)
+        self.base_url = self._submission_ws_url
+
+    SUBMISSION_WS_URL = 'https://www.ebi.ac.uk/eva/webservices/submission-ws/v1/'
+    SUBMISSION_INITIATE_PATH = 'submission/initiate'
+    SUBMISSION_UPLOADED_PATH = 'submission/{submissionId}/uploaded'
+    SUBMISSION_STATUS_PATH = 'submission/{submissionId}/status'
+
+    @property
+    def _submission_ws_url(self):
+        """Retrieve the base URL for the submission web services.
+        In order of preference from the environment variable or the hardcoded value."""
+        if os.environ.get(SUBMISSION_WS_VAR):
+            return os.environ.get(SUBMISSION_WS_VAR)
+        else:
+            return self.SUBMISSION_WS_URL
+
+    def _submission_initiate_url(self):
+        return os.path.join(self.base_url, self.SUBMISSION_INITIATE_PATH)
+
+    def _submission_uploaded_url(self, submission_id):
+        return os.path.join(self.base_url, self.SUBMISSION_UPLOADED_PATH.format(submissionId=submission_id))
+
+    def _submission_status_url(self, submission_id):
+        return os.path.join(self.base_url, self.SUBMISSION_STATUS_PATH.format(submissionId=submission_id))
+
+    def mark_submission_uploaded(self, submission_id, metadata_json):
+        response = requests.put(self._submission_uploaded_url(submission_id),
+                                headers={'Accept': 'application/json', 'Authorization': 'Bearer ' + self.auth.token},
+                                json=metadata_json)
+        response.raise_for_status()
+        return response.json()
+
+    def initiate_submission(self):
+        response = requests.post(self._submission_initiate_url(), headers={'Accept': 'application/json',
+                                                                           'Authorization': 'Bearer ' + self.auth.token})
+        response.raise_for_status()
+        return response.json()
+
+    @retry(exceptions=(HTTPError,), tries=3, delay=2, backoff=1.2, jitter=(1, 3))
+    def get_submission_status(self, submission_id):
+        response = requests.get(self._submission_status_url(submission_id))
+        response.raise_for_status()
+        return response.text