ebi-eva-common-pyutils 0.6.17__2-py3-none-any.whl

ebi_eva_common_pyutils/reference/assembly.py

@@ -0,0 +1,247 @@
+# Copyright 2019 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import urllib
+from csv import DictReader, excel_tab
+from ftplib import FTP
+import re
+from urllib import request
+
+from cached_property import cached_property
+from retry import retry
+
+from ebi_eva_common_pyutils.command_utils import run_command_with_output
+from ebi_eva_common_pyutils.logger import AppLogger
+
+
+class NCBIAssembly(AppLogger):
+    """
+    Class that represent an assembly that would originate from NCBI data
+    It takes a GCA or GCF accession and can download the assembly report and genomics fasta.
+    Using species_scientific_name and assembly_accession it create a directory structure in the provided
+    reference_directory:
+        - species_scientific_name1
+            - assembly_accession1
+            - assembly_accession2
+        - species_scientific_name2
+    the eutils_api_key is only used to retrieve additional contigs if required.
+    """
+
+    def __init__(self, assembly_accession, species_scientific_name, reference_directory, eutils_api_key=None):
+        self.check_assembly_accession_format(assembly_accession)
+        self.assembly_accession = assembly_accession
+        self.species_scientific_name = species_scientific_name
+        self.reference_directory = reference_directory
+        self.eutils_api_key = eutils_api_key
+
+    @staticmethod
+    def is_assembly_accession_format(assembly_accession):
+        if re.match(r"^GC[F|A]_\d+\.\d+$", assembly_accession) is not None:
+            return True
+        return False
+
+    @staticmethod
+    def check_assembly_accession_format(assembly_accession):
+        if not NCBIAssembly.is_assembly_accession_format(assembly_accession):
+            raise ValueError(f'Invalid assembly accession: {assembly_accession} it has to be in the form of '
+                            'GCF_XXXXXXXXX.X or GCA_XXXXXXXXX.X where X is a number')
+
+    @property
+    def assembly_directory(self):
+        assembly_directory = os.path.join(
+            self.reference_directory,  self.species_scientific_name.lower().replace(' ', '_'), self.assembly_accession
+        )
+        os.makedirs(assembly_directory, exist_ok=True),
+        return assembly_directory
+
+    @property
+    def assembly_report_path(self):
+        return os.path.join(self.assembly_directory, self.assembly_accession + '_assembly_report.txt')
+
+    @property
+    def assembly_fasta_path(self):
+        return os.path.join(self.assembly_directory, self.assembly_accession + '.fa')
+
+    @property
+    def assembly_compressed_fasta_path(self):
+        return os.path.join(self.assembly_directory, self.assembly_accession + '.fa.gz')
+
+    @retry(tries=4, delay=2, backoff=1.2, jitter=(1, 3))
+    def _download_file(self, destination_file, url):
+        self.info('Download assembly file for %s to %s', self.assembly_accession, destination_file)
+        request.urlretrieve(url, destination_file)
+        request.urlcleanup()
+
+    @cached_property
+    def _ncbi_genome_folder_url_and_content(self):
+        """
+        Internal property that retrieve and store the NCBI ftp url and content of the genome folder.
+        """
+        ftp = FTP('ftp.ncbi.nlm.nih.gov', timeout=600)
+        ftp.login('anonymous', 'anonymous')
+        genome_folder = 'genomes/all/' + '/'.join([self.assembly_accession[0:3], self.assembly_accession[4:7],
+                                                   self.assembly_accession[7:10],
+                                                   self.assembly_accession[10:13]]) + '/'
+        ftp.cwd(genome_folder)
+        all_genome_subfolders = []
+        ftp.retrlines('NLST', lambda line: all_genome_subfolders.append(line))
+
+        genome_subfolders = [folder for folder in all_genome_subfolders if folder == self.assembly_accession]
+        if len(genome_subfolders) != 1:
+            self.debug('Cannot find good match for accession folder with "%s": %s match found', self.assembly_accession, len(genome_subfolders))
+            genome_subfolders = [folder for folder in all_genome_subfolders if folder.startswith(self.assembly_accession + '_')]
+        if len(genome_subfolders) != 1:
+            self.debug('Cannot find good match for accession folder with "starting with %s_": %s match found', self.assembly_accession, len(genome_subfolders))
+            genome_subfolders = [folder for folder in all_genome_subfolders if folder.startswith(self.assembly_accession)]
+        if len(genome_subfolders) != 1:
+            self.debug('Cannot find good match for accession folder with "starting with %s": %s match found', self.assembly_accession, len(genome_subfolders))
+            genome_subfolders = [folder for folder in all_genome_subfolders if self.assembly_accession in folder]
+        if len(genome_subfolders) != 1:
+            self.debug('Cannot find good match for accession folder with "%s in name": %s match found', self.assembly_accession, len(genome_subfolders))
+            raise Exception('more than one folder matches the assembly accession: ' + str(genome_subfolders))
+        ftp.cwd(genome_subfolders[0])
+        genome_files = []
+        ftp.retrlines('NLST', lambda line: genome_files.append(line))
+        url = 'ftp://' + 'ftp.ncbi.nlm.nih.gov' + '/' + genome_folder + genome_subfolders[0]
+        ftp.close()
+        return url, genome_files
+
+    @cached_property
+    def assembly_report_url(self):
+        """
+        Search on the NCBI FTP for the assembly report file and return the full url if only one found.
+        Raise if not.
+        """
+        url, genome_files = self._ncbi_genome_folder_url_and_content
+        assembly_reports = [genome_file for genome_file in genome_files if 'assembly_report.txt' in genome_file]
+        if len(assembly_reports) != 1:
+            raise Exception('more than one file has "assembly_report" in its name: ' + str(assembly_reports))
+        return url + '/' + assembly_reports[0]
+
+    @cached_property
+    def assembly_fasta_url(self):
+        """
+        Search on the NCBI FTP for the assembly genomics fasta file and return the full url if only one found.
+        Raise if not.
+        """
+        url, genome_files = self._ncbi_genome_folder_url_and_content
+        assembly_fasta = [genome_file for genome_file in genome_files if genome_file.endswith('_genomic.fna.gz')]
+        # Remove the entries that are from genomics dna but the whole genome
+        assembly_fasta = [fasta for fasta in assembly_fasta if '_from_' not in fasta]
+        if len(assembly_fasta) > 1:
+            raise Exception('{} file found ending with "_genomic.fna.gz" in its name: {}'.format(len(assembly_fasta),
+                                                                                                 assembly_fasta))
+        return url + '/' + assembly_fasta[0]
+
+    def get_assembly_report_rows(self):
+        """Download the assembly report if it does not exist then parse it to create a generator
+        that return each row as a dict."""
+        self.download_assembly_report()
+        with open(self.assembly_report_path) as open_file:
+            headers = None
+            # Parse the assembly report file to find the header then stop
+            for line in open_file:
+                if line.lower().startswith("# sequence-name") and "sequence-role" in line.lower():
+                    headers = line.strip().split('\t')
+                    break
+            reader = DictReader(open_file, fieldnames=headers, dialect=excel_tab)
+            for record in reader:
+                yield record
+
+    def download_assembly_report(self, overwrite=False):
+        if not os.path.isfile(self.assembly_report_path) or overwrite:
+            self._download_file(self.assembly_report_path, self.assembly_report_url)
+
+    def download_assembly_fasta(self, overwrite=False):
+        if not os.path.isfile(self.assembly_fasta_path) or overwrite:
+            self._download_file(self.assembly_compressed_fasta_path, self.assembly_fasta_url)
+            run_command_with_output(
+                'Uncompress {}'.format(self.assembly_compressed_fasta_path),
+                'gunzip -f {}'.format(self.assembly_compressed_fasta_path)
+            )
+
+    def construct_fasta_from_report(self, genbank_only=False):
+        """
+        Download the assembly report if it does not exist then create the assembly fasta from the contig.
+        If the assembly already exist then it only add any missing contig.
+        """
+        written_contigs = self.get_written_contigs(self.assembly_fasta_path)
+        contig_to_append = []
+        for row in self.get_assembly_report_rows():
+            genbank_accession = row['GenBank-Accn']
+            refseq_accession = row['RefSeq-Accn']
+            relationship = row['Relationship']
+            accession = genbank_accession
+            if relationship != '=' and genbank_accession == 'na' and not genbank_only:
+                accession = refseq_accession
+            if accession in written_contigs:
+                self.debug('Accession ' + accession + ' already in the FASTA file, don\'t need to be downloaded')
+                continue
+            if not accession or accession == 'na':
+                raise ValueError('Accession {} found in report is not valid'.format(accession))
+            contig_to_append.append(self.download_contig_sequence_from_ncbi(accession))
+
+        # Now append all the new contigs to the existing fasta
+        with open(self.assembly_fasta_path, 'a+') as fasta:
+            for contig_path in contig_to_append:
+                with open(contig_path) as sequence:
+                    for line in sequence:
+                        # Check that the line is not empty
+                        if line.strip():
+                            fasta.write(line)
+                os.remove(contig_path)
+
+    def download_contig_sequence_from_ncbi(self, accession):
+        sequence_tmp_path = os.path.join(self.assembly_directory, accession + '.fa')
+        self.download_contig_from_ncbi(accession, sequence_tmp_path)
+        self.info(accession + " downloaded and added to FASTA sequence")
+        return sequence_tmp_path
+
+    @staticmethod
+    def get_written_contigs(fasta_path):
+        written_contigs = []
+        match = re.compile(r'>(.*?)\s')
+        if os.path.isfile(fasta_path):
+            with open(fasta_path, 'r') as file:
+                for line in file:
+                    written_contigs.extend(match.findall(line))
+        return written_contigs
+
+    @retry(tries=4, delay=2, backoff=1.2, jitter=(1, 3))
+    def download_contig_from_ncbi(self, contig_accession, output_file):
+        parameters = {
+            'db': 'nuccore',
+            'id': contig_accession,
+            'rettype': 'fasta',
+            'retmode': 'text',
+            'tool': 'eva',
+            'email': 'eva-dev@ebi.ac.uk'
+        }
+        if self.eutils_api_key:
+            parameters['api_key'] = self.eutils_api_key
+
+        url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?' + urllib.parse.urlencode(parameters)
+        self.info('Downloading ' + contig_accession)
+        urllib.request.urlretrieve(url, output_file)
+
+    def download_or_construct(self, genbank_only=False, overwrite=False):
+        """First download the assembly report and fasta from the FTP, then append any missing contig from
+        the assembly report to the assembly fasta."""
+        self.download_assembly_report(overwrite)
+        try:
+            self.download_assembly_fasta(overwrite)
+        except:
+            pass
+        # This will either confirm the presence of all the contig or download any one missing
+        self.construct_fasta_from_report(genbank_only)

ebi_eva_common_pyutils/reference/sequence.py

@@ -0,0 +1,101 @@
+# Copyright 2019 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import urllib
+from csv import DictReader, excel_tab
+from ftplib import FTP
+import re
+from urllib import request
+
+from cached_property import cached_property
+from retry import retry
+
+from ebi_eva_common_pyutils.command_utils import run_command_with_output
+from ebi_eva_common_pyutils.logger import AppLogger
+
+
+class NCBISequence(AppLogger):
+    """
+    Class that represent an Sequence that would originate from NCBI data
+    It takes a any Genbank or refseq accession and can download the sequence genomics fasta form.
+    Using species_scientific_name and assembly_accession it create a directory structure in the provided
+    reference_directory:
+        - species_scientific_name1
+            - assembly_accession1
+            - assembly_accession2
+        - species_scientific_name2
+    """
+
+    insdc_accession_formats = [
+        r'[A-Z][0-9]{5}\.[0-9]+',
+        r'[A-Z]{2}[0-9]{6}\.[0-9]+',
+        r'[A-Z]{2}[0-9]{8}\.[0-9]+',
+        r'[A-Z]{4}[0-9]{8}\.[0-9]+',
+        r'[A-Z]{6}[0-9]{9}\.[0-9]+'
+    ]
+
+    def __init__(self, sequence_accession, species_scientific_name, reference_directory, eutils_api_key=None):
+        self.sequence_accession = sequence_accession
+        self.species_scientific_name = species_scientific_name
+        self.reference_directory = reference_directory
+        self.eutils_api_key = eutils_api_key
+
+    @staticmethod
+    def is_genbank_accession_format(accession):
+        if any(
+                re.match(insdc_accession_format, accession)
+                for insdc_accession_format in NCBISequence.insdc_accession_formats
+        ):
+            return True
+        return False
+
+    @staticmethod
+    def check_genbank_accession_format(accession):
+        if not NCBISequence.is_genbank_accession_format(accession):
+            raise ValueError('Invalid INSDC accession: %s' % accession)
+
+    @property
+    def sequence_directory(self):
+        sequence_directory = os.path.join(
+            self.reference_directory,  self.species_scientific_name.lower().replace(' ', '_'), self.sequence_accession
+        )
+        os.makedirs(sequence_directory, exist_ok=True),
+        return sequence_directory
+
+    @property
+    def sequence_fasta_path(self):
+        return os.path.join(self.sequence_directory, self.sequence_accession + '.fa')
+
+    def download_contig_sequence_from_ncbi(self, genbank_only=True):
+        if genbank_only:
+            self.check_genbank_accession_format(self.sequence_accession)
+        self._download_contig_from_ncbi(self.sequence_accession, self.sequence_fasta_path)
+        self.info(self.sequence_fasta_path + " downloaded and added to FASTA sequence")
+
+    @retry(tries=4, delay=2, backoff=1.2, jitter=(1, 3))
+    def _download_contig_from_ncbi(self, contig_accession, output_file):
+        parameters = {
+            'db': 'nuccore',
+            'id': contig_accession,
+            'rettype': 'fasta',
+            'retmode': 'text',
+            'tool': 'eva',
+            'email': 'eva-dev@ebi.ac.uk'
+        }
+        if self.eutils_api_key:
+            parameters['api_key'] = self.eutils_api_key
+        url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?' + urllib.parse.urlencode(parameters)
+        self.info('Downloading ' + contig_accession)
+        urllib.request.urlretrieve(url, output_file)
+

ebi_eva_common_pyutils/spreadsheet/metadata_xlsx_utils.py

@@ -0,0 +1,15 @@
+import re
+
+from openpyxl.reader.excel import load_workbook
+
+
+def metadata_xlsx_version(metadata_xlsx):
+    workbook = load_workbook(metadata_xlsx)
+    try:
+        instructions_sheet = workbook['PLEASE READ FIRST']
+        xlsx_sheet_version_value = instructions_sheet[3][0].value
+        match = re.search(r'(\d+\.\d+\.\d+)', '' if xlsx_sheet_version_value is None else xlsx_sheet_version_value)
+        xlsx_version = match.group(1) if match else None
+    except (KeyError, IndexError):
+        xlsx_version = None
+    return xlsx_version

ebi_eva_common_pyutils/taxonomy/taxonomy.py

@@ -0,0 +1,60 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from ebi_eva_common_pyutils.ncbi_utils import retrieve_species_scientific_name_from_tax_id_ncbi
+from ebi_eva_common_pyutils.network_utils import json_request
+from ebi_eva_common_pyutils.logger import logging_config as log_cfg
+
+
+logger = log_cfg.get_logger(__name__)
+
+
+def get_scientific_name_from_ensembl(taxonomy_id: int) -> str:
+    ENSEMBL_REST_API_URL = "https://rest.ensembl.org/taxonomy/id/{0}?content-type=application/json".format(taxonomy_id)
+    response = json_request(ENSEMBL_REST_API_URL)
+    if "scientific_name" not in response:
+        raise Exception("Scientific name could not be found for taxonomy {0} using the Ensembl API URL: {1}"
+                        .format(taxonomy_id, ENSEMBL_REST_API_URL))
+    return response["scientific_name"]
+
+
+def normalise_taxon_scientific_name(taxon_name):
+    """
+    Match Ensembl representation
+    See Clostridium sp. SS2/1 represented as clostridium_sp_ss2_1 in
+    ftp://ftp.ensemblgenomes.org/pub/bacteria/release-48/fasta/bacteria_25_collection/clostridium_sp_ss2_1/
+    """
+    return re.sub('[^0-9a-zA-Z]+', '_', taxon_name.lower())
+
+
+def get_normalized_scientific_name_from_ensembl(taxonomy_id: int) -> str:
+    """Get the scientific name for that taxon"""
+    return normalise_taxon_scientific_name(get_scientific_name_from_ensembl(taxonomy_id))
+
+
+def get_scientific_name_from_taxonomy(taxonomy_id: int, api_key: str=None) -> str:
+    """
+    Search for a species scientific name based on the taxonomy id.
+    Will first attempt to retrieve from Ensembl and then NCBI, if not found returns None.
+    """
+    try:
+        species_name = get_scientific_name_from_ensembl(taxonomy_id)
+    except Exception:
+        logger.warning("Failed to retrieve scientific name in Ensembl for taxonomy id {0}".format(taxonomy_id))
+        species_name = None
+    if not species_name:
+        species_name = retrieve_species_scientific_name_from_tax_id_ncbi(taxonomy_id, api_key=api_key)
+    return species_name

ebi_eva_common_pyutils/variation/contig_utils.py

@@ -0,0 +1,113 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import requests
+from retry import retry
+
+
+# TODO: Might be a good idea to re-visit this after a production implementation
+#  of the contig-alias resolution project is available
+@retry(tries=10, delay=5, backoff=1.2, jitter=(1, 3))
+def resolve_contig_accession_to_chromosome_name(contig_accession, line_limit=100):
+    """
+    Given a Genbank contig accession, get the corresponding chromosome name from the ENA Text API
+    which returns results in a EMBL Flatfile format
+
+    :param contig_accession: Genbank contig accession (ex:  CM003032.1)
+    :param line_limit: number of lines to parse in the EMBL Flatfile result to find the chromosome before giving up
+    :return: Chromosome name (ex: 12 when given an accession CM003032.1)
+    """
+    ENA_TEXT_API_URL = "https://www.ebi.ac.uk/ena/browser/api/text/{0}?lineLimit={1}&annotationOnly=true"
+    response = requests.get(ENA_TEXT_API_URL.format(contig_accession, line_limit))
+    response_lines = response.content.decode("utf-8").split("\n")
+    num_lines = len(response_lines)
+
+    features_section_found, source_line_found = False, False
+    chosen_response = []
+    line_index = 0
+    # Look for the "source" feature under the "Features" section in the text response
+    while line_index < num_lines:
+        line = response_lines[line_index]
+        if not (features_section_found or line.lower().startswith("fh   key")):
+            line_index += 1
+            continue
+        features_section_found = True
+        # Based on "Data item positions" described here, http://www.insdc.org/files/feature_table.html#3.4.2
+        # the sixth character represents the start of the feature key
+        if not (source_line_found or line[5:].lower().startswith("source")):
+            line_index += 1
+            continue
+        source_line_found = True
+        if line[21:].startswith("/"):
+            assembled_line = line.strip()
+            line_index += 1
+            # Assemble text spread across multiple lines until
+            # we hit the next qualifier (starts with /) or the next section
+            while line_index < num_lines and \
+                    not (response_lines[line_index][21:].startswith("/")
+                         or response_lines[line_index][5:6].strip() != ''):
+                line = response_lines[line_index]
+                assembled_line += " " + line[21:].strip()
+                line_index += 1
+
+            # Fall back to organelle in case of MT/Chloroplast accessions
+            # and the reference notes in case of Linkage Group molecules
+            chosen_response = re.findall('.*/chromosome=".+"', assembled_line) or \
+                              re.findall('.*/organelle=".+"', assembled_line) or \
+                              re.findall('.*/note=".+"', assembled_line)
+
+            # If we have a response to give, no need to continue further
+            # If the sixth character is not empty, we have reached the next feature, so no need to continue further
+            if chosen_response or line[5:6].strip() != '':
+                break
+        else:
+            line_index += 1
+
+    if not chosen_response:
+        return ""
+
+    return str.split(chosen_response[0], '"')[1].strip()
+
+
+def is_wgs_accession_format(contig_accession):
+    """
+    Check if a Genbank contig is part of WGS (Whole Genome Shotgun) sequence
+
+    :param contig_accession: Genbank contig accession (ex:  CM003032.1)
+    :return: True if the provided contig is in the WGS format
+    """
+    wgs_prefix = contig_accession[:4]
+    wgs_numeric_suffix = contig_accession[4:].replace(".", "")
+    return str.isalpha(wgs_prefix) and str.isnumeric(wgs_numeric_suffix)
+
+
+def get_chromosome_name_for_contig_accession(contig_accession):
+    """
+    Given a Genbank contig accession, get the corresponding chromosome name
+
+    :param contig_accession: Genbank contig accession (ex:  CM003032.1)
+    :return: Chromosome name (ex: 12 when given an accession CM003032.1)
+    """
+
+    # Don't bother calling the ENA web service to get the chromosome number if the accession is a WGS accession
+    # since the API will proceed to download the entire WGS dataset which can be in hundreds of MBs or even GBs
+    # See https://www.ebi.ac.uk/ena/browser/api/text/AABR07050911.1?lineLimit=100&annotationOnly=true for example
+    if is_wgs_accession_format(contig_accession):
+        return None
+
+    return \
+        resolve_contig_accession_to_chromosome_name(contig_accession, 1000) or \
+        resolve_contig_accession_to_chromosome_name(contig_accession, 10000) or \
+        resolve_contig_accession_to_chromosome_name(contig_accession, 100000)

ebi_eva_common_pyutils-0.6.17.data/scripts/archive_directory.py

@@ -0,0 +1,114 @@
+#!python
+import gzip
+import shutil
+import tarfile
+import os.path
+from argparse import ArgumentParser
+
+from ebi_eva_common_pyutils.logger import logging_config
+from retry import retry
+
+logger = logging_config.get_logger(__name__)
+
+
+def make_tarfile(output_filename, source_dir):
+    logger.info(f'Create Final Tar file {output_filename}.')
+    with tarfile.open(output_filename, "w") as tar:
+        tar.add(source_dir, arcname=os.path.basename(source_dir))
+    file_stats = os.stat(output_filename)
+    logger.info(f'{output_filename} completed. File Size in Bytes is {file_stats.st_size}')
+
+
+def is_compressed(file_path):
+    compressed_ext = ['.gz', '.zip', '.bz', '.tbi', '.csi']
+    for ext in compressed_ext:
+        if file_path.lower().endswith(ext):
+            return True
+    return False
+
+
+@retry(tries=5, delay=3, backoff=2, logger=logger)
+def retriable_compress(src_file_path, dest_file_path):
+    MEG = 2 ** 20
+    with open(src_file_path, 'rb') as f_in:
+        with gzip.open(dest_file_path, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out, length=16 * MEG)
+
+
+def matches(name, patterns):
+    return any((pattern for pattern in patterns if pattern in name))
+
+
+@retry(tries=5, delay=3, backoff=2, logger=logger)
+def retriable_remove(path):
+    shutil.rmtree(path)
+
+
+@retry(tries=5, delay=3, backoff=2, logger=logger)
+def retryable_copy(src_file_path, dest_file_path, **kwargs):
+    if os.path.exists(dest_file_path):
+        os.remove(dest_file_path)
+    shutil.copyfile(src_file_path, dest_file_path, **kwargs)
+
+
+def archive_directory(source_dir, scratch_dir, destination_dir, filter_patterns=None):
+    """
+    Archive a directory by copying the data it contains to a scratch directory compressing all files that are not
+    already compressed. Then it create a tar file to the destination_dir under the name of the original directory.
+    """
+    source_dir_name = os.path.basename(source_dir)
+    logger.info(f'Archive {source_dir_name} from {source_dir}')
+    parent_source_dir = os.path.dirname(source_dir)
+    for base, dirs, files in os.walk(source_dir, topdown=True, followlinks=False):
+        # Filter the downstream directory to
+        filtered_dir = []
+        for d in dirs:
+            if matches(d, filter_patterns):
+                logger.info(f'Ignore directory {d} because of filters: {filter_patterns}')
+            else:
+                filtered_dir.append(d)
+        # modify dirs in place
+        dirs[:] = filtered_dir
+        src_basename = os.path.relpath(base, parent_source_dir)
+        scratch_dest_dir = os.path.join(scratch_dir, src_basename)
+        os.makedirs(scratch_dest_dir, exist_ok=True)
+        for fname in files:
+            src_file_path = os.path.join(base, fname)
+            dest_file_path = os.path.join(scratch_dest_dir, fname)
+            if matches(fname, filter_patterns):
+                logger.info(f'Ignore file {src_file_path} because of filters: {filter_patterns}')
+                continue
+            if os.path.islink(src_file_path) or is_compressed(src_file_path):
+                logger.info(f'Copy {src_file_path}')
+                retryable_copy(src_file_path, dest_file_path, follow_symlinks=False)
+            else:
+                logger.info(f'Compress {src_file_path}')
+                retriable_compress(src_file_path, dest_file_path + '.gz')
+    final_tar_file = os.path.join(destination_dir, source_dir_name + '.tar')
+    scratch_dir_archived = os.path.join(scratch_dir, source_dir_name)
+    make_tarfile(final_tar_file, scratch_dir_archived)
+    logger.info(f'Delete scratch folder {scratch_dir_archived}.')
+    retriable_remove(scratch_dir_archived)
+    logger.info(f'Scratch folder {scratch_dir_archived} deleted.')
+
+
+def main():
+    parser = ArgumentParser(description='Archive a directory by copying the data it contains to a scratch directory '
+                                        'compressing all files that are not already compressed. Then it create a tar '
+                                        'file to the destination_dir under the name of the original directory.')
+    parser.add_argument('--source_dir', required=True, type=str,
+                        help='base directory you want to archive. All sub directories will be included.')
+    parser.add_argument('--destination_dir', required=True, type=str,
+                        help='Directory where the archive should be placed at the end of the process.')
+    parser.add_argument('--scratch_dir', required=True, type=str,
+                        help='Directory where the archive will be constructed.')
+    parser.add_argument('--filter_patterns', type=str, nargs='*', default=[],
+                        help='keyword found in file and directory names that not be included to the archive.')
+    args = parser.parse_args()
+
+    logging_config.add_stdout_handler()
+    archive_directory(args.source_dir, args.scratch_dir,  args.destination_dir, args.filter_patterns)
+
+
+if __name__ == '__main__':
+    main()