ebi-eva-common-pyutils 0.6.15__1-py3-none-any.whl

ebi_eva_internal_pyutils/metadata_utils.py

@@ -0,0 +1,288 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import datetime
+import re
+from urllib.parse import urlsplit
+
+import psycopg2
+
+from ebi_eva_common_pyutils.assembly_utils import is_patch_assembly
+from ebi_eva_internal_pyutils.config_utils import get_metadata_creds_for_profile
+from ebi_eva_common_pyutils.ena_utils import get_scientific_name_and_common_name
+from ebi_eva_common_pyutils.logger import logging_config
+from ebi_eva_common_pyutils.ncbi_utils import get_ncbi_assembly_name_from_term
+from ebi_eva_internal_pyutils.pg_utils import get_result_cursor, get_all_results_for_query, execute_query
+from ebi_eva_common_pyutils.taxonomy.taxonomy import get_scientific_name_from_ensembl
+
+logger = logging_config.get_logger(__name__)
+SUPPORTED_ASSEMBLY_TRACKER_TABLE = "evapro.supported_assembly_tracker"
+
+
+def get_metadata_connection_handle(profile, settings_xml_file):
+    pg_url, pg_user, pg_pass = get_metadata_creds_for_profile(profile, settings_xml_file)
+    return psycopg2.connect(urlsplit(pg_url).path, user=pg_user, password=pg_pass)
+
+
+def get_db_conn_for_species(species_db_info):
+    db_name = "dbsnp_{0}".format(species_db_info["dbsnp_build"])
+    pg_conn = psycopg2.connect("dbname='{0}' user='{1}' host='{2}'  port={3}".
+                               format(db_name, "dbsnp", species_db_info["pg_host"], species_db_info["pg_port"]))
+    return pg_conn
+
+
+def get_species_info(metadata_connection_handle, dbsnp_species_name="all"):
+    get_species_info_query = "SELECT DISTINCT database_name, scientific_name, dbsnp_build, pg_host, pg_port " \
+                             "FROM dbsnp_ensembl_species.import_progress a " \
+                             "JOIN dbsnp_ensembl_species.dbsnp_build_instance b " \
+                             "ON b.dbsnp_build = a.ebi_pg_dbsnp_build "
+    if dbsnp_species_name != "all":
+        get_species_info_query += "where database_name = '{0}' ".format(dbsnp_species_name)
+    get_species_info_query += "order by database_name"
+
+    pg_cursor = get_result_cursor(metadata_connection_handle, get_species_info_query)
+    species_set = [{"database_name": result[0], "scientific_name": result[1], "dbsnp_build":result[2],
+                    "pg_host":result[3], "pg_port":result[4]}
+                   for result in pg_cursor.fetchall()]
+    pg_cursor.close()
+    return species_set
+
+
+# Get connection information for each Postgres instance of the dbSNP mirror
+def get_dbsnp_mirror_db_info(pg_metadata_dbname, pg_metadata_user, pg_metadata_host):
+    with psycopg2.connect("dbname='{0}' user='{1}' host='{2}'".format(pg_metadata_dbname, pg_metadata_user,
+                                                                      pg_metadata_host)) as pg_conn:
+        dbsnp_mirror_db_info_query = "SELECT * FROM dbsnp_ensembl_species.dbsnp_build_instance"
+        dbsnp_mirror_db_info = [{"dbsnp_build": result[0], "pg_host": result[1], "pg_port": result[2]}
+                                for result in get_all_results_for_query(pg_conn, dbsnp_mirror_db_info_query)]
+    return dbsnp_mirror_db_info
+
+
+def get_taxonomy_code_from_metadata(metadata_connection_handle, taxonomy):
+    """
+    Retrieve an existing taxonomy code registered in the metadata database.
+    """
+    query = f"SELECT DISTINCT t.taxonomy_code FROM taxonomy t WHERE t.taxonomy_id = {taxonomy}"
+    rows = get_all_results_for_query(metadata_connection_handle, query)
+    if len(rows) == 0:
+        return None
+    elif len(rows) > 1:
+        options = ', '.join(rows)
+        raise ValueError(f'More than one possible code for taxonomy {taxonomy} found: {options}')
+    return rows[0][0]
+
+
+def get_assembly_code_from_metadata(metadata_connection_handle, assembly):
+    """
+    Retrieve an existing assembly code registered in the metadata database.
+    """
+    query = f"SELECT DISTINCT assembly_code FROM assembly WHERE assembly_accession='{assembly}'"
+    rows = get_all_results_for_query(metadata_connection_handle, query)
+    if len(rows) == 0:
+        return None
+    elif len(rows) > 1:
+        options = ', '.join([row for row, in rows])
+        raise ValueError(f'More than one possible code for assembly {assembly} found: {options}')
+    return rows[0][0]
+
+
+def build_variant_warehouse_database_name(taxonomy_code, assembly_code):
+    if taxonomy_code and assembly_code:
+        return f'eva_{taxonomy_code}_{assembly_code}'
+    return None
+
+
+def resolve_existing_variant_warehouse_db_name(metadata_connection_handle, assembly, taxonomy):
+    """
+    Retrieve an existing database name by combining the taxonomy_code and assembly code registered in the metadata
+    database.
+    """
+    return build_variant_warehouse_database_name(
+        get_taxonomy_code_from_metadata(metadata_connection_handle, taxonomy),
+        get_assembly_code_from_metadata(metadata_connection_handle, assembly)
+    )
+
+
+# For backward compatibility
+get_variant_warehouse_db_name_from_assembly_and_taxonomy = resolve_existing_variant_warehouse_db_name
+
+
+def get_assembly_code(metadata_connection_handle, assembly, ncbi_api_key=None):
+    assembly_code = get_assembly_code_from_metadata(metadata_connection_handle, assembly)
+    if not assembly_code:
+        assembly_name = get_ncbi_assembly_name_from_term(assembly, api_key=ncbi_api_key)
+        # If the assembly is a patch assembly ex: GRCh37.p8, drop the trailing patch i.e., just return grch37
+        if is_patch_assembly(assembly):
+            assembly_name = re.sub('\\.p[0-9]+$', '', assembly_name.lower())
+        assembly_code = re.sub('[^0-9a-zA-Z]+', '', assembly_name.lower())
+    return assembly_code
+
+
+def build_taxonomy_code(scientific_name):
+    """Given a scientific name like "Zea mays", the corresponding taxonomy code should be zmays"""
+    return scientific_name[0].lower() + re.sub('[^0-9a-zA-Z]+', '', ''.join(scientific_name.split()[1:])).lower()
+
+
+def get_taxonomy_code(metadata_connection_handle, taxonomy):
+    taxonomy_code = get_taxonomy_code_from_metadata(metadata_connection_handle, taxonomy)
+    if not taxonomy_code:
+        scientific_name = get_scientific_name_from_ensembl(taxonomy)
+        taxonomy_code = build_taxonomy_code(scientific_name)
+    return taxonomy_code
+
+
+def resolve_variant_warehouse_db_name(metadata_connection_handle, assembly, taxonomy, ncbi_api_key=None):
+    """
+    Retrieve the database name for this taxonomy/assembly pair whether it exists or not.
+    It will use existing taxonomy code or assembly code if available in the metadata database.
+    """
+    taxonomy_code = get_taxonomy_code(metadata_connection_handle, taxonomy)
+    assembly_code = get_assembly_code(metadata_connection_handle, assembly, ncbi_api_key=ncbi_api_key)
+    return build_variant_warehouse_database_name(taxonomy_code, assembly_code)
+
+
+def insert_new_assembly_and_taxonomy(metadata_connection_handle, assembly_accession, taxonomy_id, eva_species_name=None,
+                                     in_accessioning=True, ncbi_api_key=None):
+    """
+    This script adds new assemblies and taxonomies to EVAPRO.
+    You can also add the assembly with a different taxonomy if you provide the
+    taxonomy parameters. Example taxonomy page:
+    https://www.ebi.ac.uk/ena/data/view/Taxon:9031
+
+    :param assembly_accession: Assembly accession (Example: GCA_000002315.3)
+    :param metadata_connection_handle: Metadata DB connection
+    :param taxonomy_id: Taxonomy id (Example: 9031)
+    :param eva_species_name: EVA species name (Example: chicken).
+        Not required if the taxonomy exists or ENA has a common name available.
+    :param in_accessioning: Flag that this assembly is in the accessioning data store.
+    """
+    # check if assembly is already in EVAPRO, adding it if not
+    assembly_set_id = get_assembly_set_from_metadata(metadata_connection_handle, taxonomy_id, assembly_accession)
+    if assembly_set_id is None:
+        assembly_name = get_ncbi_assembly_name_from_term(assembly_accession, api_key=ncbi_api_key)
+        ensure_taxonomy_is_in_evapro(metadata_connection_handle, taxonomy_id, eva_species_name)
+        assembly_code = get_assembly_code(metadata_connection_handle, assembly_accession)
+        insert_assembly_in_evapro(metadata_connection_handle, taxonomy_id, assembly_accession, assembly_name, assembly_code)
+
+    update_accessioning_status(metadata_connection_handle, assembly_accession, in_accessioning)
+    metadata_connection_handle.commit()
+
+
+def ensure_taxonomy_is_in_evapro(metadata_connection_handle, taxonomy, eva_species_name=None):
+    if is_taxonomy_in_evapro(metadata_connection_handle, taxonomy):
+        logger.debug('Taxonomy {} is already in the database'.format(taxonomy))
+    else:
+        logger.info("Taxonomy {} not present in EVAPRO. Adding taxonomy ...".format(taxonomy))
+        scientific_name, common_name = get_scientific_name_and_common_name(taxonomy)
+        taxonomy_code = build_taxonomy_code(scientific_name)
+        # If a common name cannot be found then we should  use the scientific name
+        eva_species_name = eva_species_name or common_name or scientific_name
+        insert_taxonomy(metadata_connection_handle, taxonomy, scientific_name, common_name, taxonomy_code, eva_species_name)
+
+
+def insert_assembly_in_evapro(metadata_connection_handle, taxonomy_id, assembly_accession, assembly_name, assembly_code):
+    cur = metadata_connection_handle.cursor()
+    cur.execute('INSERT INTO evapro.assembly_set(taxonomy_id, assembly_name, assembly_code) VALUES (%s, %s, %s)',
+                (taxonomy_id, assembly_name, assembly_code))
+
+    # get the assembly_set_id that was autogenerated in the row that we just inserted in assembly_set
+    assembly_set_id = get_all_results_for_query(metadata_connection_handle,
+                                    'SELECT assembly_set_id FROM evapro.assembly_set '
+                                    'WHERE taxonomy_id={} and assembly_name=\'{}\' and assembly_code=\'{}\''
+                                                .format(taxonomy_id, assembly_name, assembly_code))[0][0]
+
+    assembly_chain = assembly_accession.split('.')[0]
+    assembly_version = assembly_accession.split('.')[1]
+    cur.execute('INSERT INTO evapro.accessioned_assembly('
+                'assembly_set_id, assembly_accession, assembly_chain, assembly_version) VALUES (%s,%s,%s,%s)',
+                (assembly_set_id, assembly_accession, assembly_chain, assembly_version))
+
+    logger.info('New assembly added with assembly_set_id: {0}'.format(assembly_set_id))
+    return assembly_set_id
+
+
+def update_accessioning_status(metadata_connection_handle, assembly_accession, in_accessioning_flag):
+    cur = metadata_connection_handle.cursor()
+    # Only insert assembly accessions which are NOT already in the assembly_accessioning_store_status table
+    assembly_accessioning_store_insert_query = "INSERT INTO evapro.assembly_accessioning_store_status " \
+                                               "SELECT * FROM (SELECT " \
+                                               "cast('{0}' as text) as assembly_accession" \
+                                               ", cast('{1}' as boolean) as loaded) temp " \
+                                               "WHERE assembly_accession NOT IN " \
+                                               "(SELECT assembly_accession FROM " \
+                                               "evapro.assembly_accessioning_store_status)" \
+                                               .format(assembly_accession, in_accessioning_flag)
+    cur.execute(assembly_accessioning_store_insert_query)
+
+
+def get_assembly_set_from_metadata(metadata_connection_handle, taxonomy, assembly_accession):
+    query = (f"SELECT acc.assembly_set_id "
+             f"FROM evapro.accessioned_assembly acc "
+             f"JOIN assembly_set asm on acc.assembly_set_id = asm.assembly_set_id "
+             f"WHERE assembly_accession='{assembly_accession}' AND taxonomy_id={taxonomy}")
+    rows = get_all_results_for_query(metadata_connection_handle, query)
+
+    if len(rows) == 1:
+        return rows[0][0]
+    elif len(rows) == 0:
+        return None
+    else:
+        raise ValueError('Inconsistent database state: several assembly_set_ids for the same taxonomy ({}) and '
+                         'assembly accession ({}): {}'.format(taxonomy, assembly_accession, rows))
+
+
+def is_taxonomy_in_evapro(metadata_connection_handle, taxonomy_id):
+    taxonomy_query = 'SELECT taxonomy_id FROM evapro.taxonomy WHERE taxonomy_id={}'.format(taxonomy_id)
+    taxonomy_ids_in_evapro = get_all_results_for_query(metadata_connection_handle, taxonomy_query)
+    return len(taxonomy_ids_in_evapro) > 0
+
+
+def insert_taxonomy(metadata_connection_handle, taxonomy_id, scientific_name, common_name, taxonomy_code, eva_species_name):
+    if taxonomy_code is None or eva_species_name is None:
+        raise ValueError('Error: taxonomy code ({}) and EVA taxonomy name ({}) are required '
+                         'for inserting a taxonomy'.format(taxonomy_code, eva_species_name))
+    cur = metadata_connection_handle.cursor()
+    cur.execute('INSERT INTO evapro.taxonomy(taxonomy_id, common_name, scientific_name, taxonomy_code, eva_name) '
+                'VALUES (%s, %s, %s, %s, %s)',
+                (taxonomy_id, common_name, scientific_name, taxonomy_code, eva_species_name))
+    logger.info('New taxonomy {} added'.format(taxonomy_id))
+
+
+def add_to_supported_assemblies(metadata_connection_handle, source_of_assembly: str, target_assembly: str,
+                                taxonomy_id: int):
+    today = datetime.date.today().strftime('%Y-%m-%d')
+    # First check if the current assembly is already target - if so don't do anything
+    current_query = (
+        f"SELECT assembly_id FROM {SUPPORTED_ASSEMBLY_TRACKER_TABLE} "
+        f"WHERE taxonomy_id={taxonomy_id} AND current=true;"
+    )
+    results = get_all_results_for_query(metadata_connection_handle, current_query)
+    if len(results) > 0 and results[0][0] == target_assembly:
+        logger.warning(f'Current assembly for taxonomy {taxonomy_id} is already {target_assembly}!')
+        return
+
+    # Deprecate the last current assembly
+    update_query = (
+        f"UPDATE {SUPPORTED_ASSEMBLY_TRACKER_TABLE} "
+        f"SET current=false, end_date='{today}' "
+        f"WHERE taxonomy_id={taxonomy_id} AND current=true;"
+    )
+    execute_query(metadata_connection_handle, update_query)
+
+    # Then insert the new assembly
+    insert_query = (
+        f"INSERT INTO {SUPPORTED_ASSEMBLY_TRACKER_TABLE} "
+        f"(taxonomy_id, source, assembly_id, current, start_date) "
+        f"VALUES({taxonomy_id}, '{source_of_assembly}', '{target_assembly}', true, '{today}');"
+    )
+    execute_query(metadata_connection_handle, insert_query)

ebi_eva_internal_pyutils/mongo_utils.py

@@ -0,0 +1,71 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pymongo
+from urllib.parse import quote_plus
+
+from pymongo import ReadPreference
+
+from ebi_eva_common_pyutils.command_utils import run_command_with_output
+from ebi_eva_common_pyutils.common_utils import merge_two_dicts
+from ebi_eva_internal_pyutils.config_utils import get_mongo_uri_for_eva_profile, get_mongo_creds_for_profile
+
+
+class MongoConfig:
+    parameters = None
+
+    def __init__(self, **kwargs):
+        self.parameters = kwargs
+        if "port" not in kwargs:
+            self.parameters["port"] = 27017
+        if "host" not in kwargs:
+            self.parameters["host"] = "localhost"
+
+
+def get_mongo_connection_handle(profile: str, settings_xml_file: str,
+                                read_concern: str = "majority",
+                                read_preference: ReadPreference = ReadPreference.PRIMARY,
+                                write_concern: str = "majority") -> pymongo.MongoClient:
+    mongo_connection_uri = get_mongo_uri_for_eva_profile(profile, settings_xml_file)
+    return pymongo.MongoClient(mongo_connection_uri,
+                               readConcernLevel=read_concern,
+                               read_preference=read_preference,
+                               w=write_concern)
+
+
+def get_primary_mongo_connection_handle(profile: str, settings_xml_file: str,
+                                        read_concern: str = "majority",
+                                        read_preference: ReadPreference = ReadPreference.PRIMARY,
+                                        write_concern: str = "majority") -> pymongo.MongoClient:
+    host, username, password = get_mongo_creds_for_profile(profile, settings_xml_file)
+    mongo_connection_uri = "mongodb://{0}:{1}@{2}:{3}/{4}".format(username, quote_plus(password), host,
+                                                                  27017, "admin")
+    return pymongo.MongoClient(mongo_connection_uri,
+                               readConcernLevel=read_concern,
+                               read_preference=read_preference,
+                               w=write_concern)
+
+
+def copy_db_with_config(mongo_source_config: MongoConfig, mongo_destination_config: MongoConfig, mongodump_args: dict,
+                        mongorestore_args: dict):
+    copy_db(merge_two_dicts(mongo_source_config.parameters, mongodump_args),
+            merge_two_dicts(mongo_destination_config.parameters, mongorestore_args))
+
+
+def copy_db(mongodump_args: dict, mongorestore_args: dict):
+    mongodump_args_str = " ".join(["--{0} {1}".format(key, value) for key, value in mongodump_args.items()])
+    mongorestore_args_str = " ".join(["--{0} {1}".format(key, value) for key, value in mongorestore_args.items()])
+    run_command_with_output("Running mongodump", "mongodump " + mongodump_args_str, log_error_stream_to_output=True)
+    run_command_with_output("Running mongorestore", "mongorestore " + mongorestore_args_str,
+                            log_error_stream_to_output=True)

ebi_eva_internal_pyutils/mongodb/__init__.py

@@ -0,0 +1,3 @@
+from ebi_eva_internal_pyutils.mongodb.mongo_database import MongoDatabase
+
+

ebi_eva_internal_pyutils/mongodb/mongo_database.py

@@ -0,0 +1,170 @@
+# Copyright 2021 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import subprocess
+
+from cached_property import cached_property
+from pymongo import MongoClient, uri_parser
+
+from ebi_eva_common_pyutils.command_utils import run_command_with_output
+from ebi_eva_common_pyutils.logger import AppLogger
+
+
+class MongoDatabase(AppLogger):
+    def __init__(self, uri: str, secrets_file: str = None, db_name: str = "admin"):
+        self.uri = uri
+        self.secrets_file = secrets_file
+        self.db_name = db_name
+
+    @cached_property
+    def uri_with_db_name(self):
+        """
+        Return URI with the database name substituted
+        ex:
+        If the URI is mongodb://user@localhost:27017/admin and database name is eva_fcatus_90,
+        then the URI with the database name will be mongodb://user@localhost:27017/eva_fcatus_90?authSource=admin
+        """
+        if self.db_name == "admin":
+            return self.uri
+        uri_components = uri_parser.parse_uri(self.uri)
+        username_component = f"{uri_components['username']}@" if uri_components['username'] else ""
+        # Hack needed to log in to a different DB but retain authentication source
+        # See https://docs.mongodb.com/v4.0/reference/connection-string/#records-database and https://docs.mongodb.com/v4.0/reference/connection-string/#urioption.authSource
+        uri_with_db_name = f"mongodb://{username_component}" + \
+                           ",".join([node + ':' + str(port) for node, port in uri_components['nodelist']]) + \
+                           f"/{self.db_name}"
+        uri_with_db_name += f"?authSource=" \
+                            f"{uri_components['options'].get('authSource', 'admin')}" if self.secrets_file else ""
+        return uri_with_db_name
+
+    @cached_property
+    def mongo_handle(self):
+        if self.secrets_file:
+            with open(self.secrets_file) as secrets_file_handle:
+                mongo_password = secrets_file_handle.read().strip()
+            return MongoClient(self.uri, password=mongo_password)
+        else:
+            return MongoClient(self.uri)
+
+    def __del__(self):
+        self.mongo_handle.close()
+
+    def _get_optional_secrets_file_stdin(self):
+        # Mongodump and restore tools are notorious in displaying clear text passwords
+        # in commands - See https://jira.mongodb.org/browse/TOOLS-1020
+        # A secrets file can be provided to work around it
+        if self.secrets_file:
+            return f" < {self.secrets_file}"
+        return ""
+
+    def drop(self):
+        self.mongo_handle.drop_database(self.db_name)
+
+    def get_collection_names(self):
+        return self.mongo_handle[self.db_name].list_collection_names()
+
+    def get_indexes(self):
+        collection_index_map = {}
+        for collection_name in self.get_collection_names():
+            collection_index_map[collection_name] = self.mongo_handle[self.db_name][collection_name].index_information()
+        return collection_index_map
+
+    def create_index_on_collections(self, collection_index_map):
+        collection_index_map_copy = copy.deepcopy(collection_index_map)
+        for collection_name, index_info_map in collection_index_map_copy.items():
+            # Copy indexes from one collection to another See https://stackoverflow.com/a/51445278
+            for name, index_info in index_info_map.items():
+                index_keys = index_info['key']
+                del (index_info['ns'])
+                del (index_info['v'])
+                del (index_info['key'])
+                if 'background' in index_info:
+                    del (index_info['background'])
+                # Due to https://jira.mongodb.org/browse/SERVER-11064
+                # pre-v3.2 index sort indicators could allow for floats like 1.0
+                # ex: (_id, 1.0) for ascending index on _id or (_id, -1.0) for descending index on _id
+                # Since validation is stricter in database versions newer than v3.3.1, cast sort indicators to int
+                for i, _ in enumerate(index_keys):
+                    index_keys[i] = (index_keys[i][0], int(index_keys[i][1]))
+                self.mongo_handle[self.db_name][collection_name].create_index(index_keys, name=name, **index_info)
+
+    def enable_sharding(self):
+        self.mongo_handle.admin.command({"enableSharding": self.db_name})
+
+    def shard_collections(self, collections_shard_key_map, collections_to_shard):
+        for collection_name in collections_to_shard:
+            shard_key, shard_key_uniqueness_flag = collections_shard_key_map.get(collection_name, (["_id"], True))
+            # Shard key representation in the format {"key1": 1, "key2": 1}
+            shard_key_repr = "{{{0}}}".format(",".join([f'"{attribute}": 1' for attribute in shard_key]))
+            shard_collection_command = f'sh.shardCollection(' \
+                                       f'"{self.db_name}.{collection_name}", ' \
+                                       f'{shard_key_repr}, {str(shard_key_uniqueness_flag).lower()})'
+            sharding_command = f"mongosh --eval '{shard_collection_command}' {self.uri} "
+            sharding_command += self._get_optional_secrets_file_stdin()
+            run_command_with_output(f"Sharding collection {collection_name} in the database {self.uri_with_db_name} "
+                                    f"with key {shard_key_repr}...", sharding_command,
+                                    log_error_stream_to_output=True, )
+
+    def dump_data(self, dump_dir, mongodump_args=None):
+        mongodump_args = " ".join([f"--{arg} {val}"
+                                   for arg, val in mongodump_args.items()]) if mongodump_args else ""
+        mongodump_command = f"mongodump --uri {self.uri_with_db_name}  --out {dump_dir} {mongodump_args}" + \
+                            self._get_optional_secrets_file_stdin()
+        try:
+            run_command_with_output("mongodump", mongodump_command, log_error_stream_to_output=True)
+        except subprocess.CalledProcessError as ex:
+            raise Exception("mongodump failed! HINT: Did you forget to provide a secrets file for authentication?")
+
+    def archive_data(self, archive_dir, archive_name="archive", mongodump_args=None):
+        mongodump_args = " ".join([f"--{arg} {val}"
+                                   for arg, val in mongodump_args.items()]) if mongodump_args else ""
+        mongodump_command = f"mongodump --uri {self.uri_with_db_name}  --archive={archive_dir}/{archive_name} {mongodump_args}" + \
+                            self._get_optional_secrets_file_stdin()
+        try:
+            run_command_with_output("mongodump", mongodump_command, log_error_stream_to_output=True)
+        except subprocess.CalledProcessError as ex:
+            raise Exception("mongodump failed! HINT: Did you forget to provide a secrets file for authentication?")
+
+    def restore_data(self, dump_dir, mongorestore_args=None):
+        mongorestore_args = " ".join([f"--{arg} {val}"
+                                      for arg, val in mongorestore_args.items()]) if mongorestore_args else ""
+        mongorestore_command = f"mongorestore --uri {self.uri_with_db_name} " \
+                               f"{mongorestore_args} " \
+                               f"--dir {dump_dir} "
+        mongorestore_command += self._get_optional_secrets_file_stdin()
+        try:
+            run_command_with_output("mongorestore", mongorestore_command, log_error_stream_to_output=True)
+        except subprocess.CalledProcessError as ex:
+            raise Exception("mongorestore failed! HINT: Did you forget to provide a secrets file for authentication?")
+
+    def export_data(self, export_directory, mongoexport_args=None):
+        mongoexport_args = " ".join([f"--{arg} {val}"
+                                     for arg, val in mongoexport_args.items()]) if mongoexport_args else ""
+        mongoexport_command = f"mongoexport --uri {self.uri_with_db_name}  --out {export_directory} {mongoexport_args}" + \
+                              self._get_optional_secrets_file_stdin()
+        try:
+            run_command_with_output("mongoexport", mongoexport_command, log_error_stream_to_output=True)
+        except subprocess.CalledProcessError as ex:
+            raise Exception("mongoexport failed! HINT: Did you forget to provide a secrets file for authentication?")
+
+    def import_data(self, coll_file_loc, mongoimport_args=None):
+        mongoimport_args = " ".join([f"--{arg} {val}"
+                                     for arg, val in mongoimport_args.items()]) if mongoimport_args else ""
+        mongoimport_command = f"mongoimport --uri {self.uri_with_db_name}  --file {coll_file_loc} {mongoimport_args}" + \
+                              self._get_optional_secrets_file_stdin()
+        try:
+            run_command_with_output("mongoimport", mongoimport_command, log_error_stream_to_output=True)
+        except subprocess.CalledProcessError as ex:
+            raise Exception("mongoexport failed! HINT: Did you forget to provide a secrets file for authentication?")

ebi_eva_internal_pyutils/nextflow/__init__.py

@@ -0,0 +1 @@
+from ebi_eva_internal_pyutils.nextflow.nextflow_pipeline import LinearNextFlowPipeline, NextFlowPipeline, NextFlowProcess