ebi-eva-common-pyutils 0.6.15__1-py3-none-any.whl

ebi_eva_internal_pyutils/nextflow/nextflow_pipeline.py

@@ -0,0 +1,195 @@
+# Copyright 2021 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Rationale for a Nextflow pipeline abstraction
+# ---------------------------------------------
+# Dynamic pipeline generation
+# Abstraction to represent process dependencies
+# Unit testability of individual steps without scattering logic between Python and Nextflow
+# Ability to combine pipelines
+
+import networkx as nx
+import os
+from typing import List, Dict, Union
+
+from ebi_eva_common_pyutils.logger import AppLogger
+from ebi_eva_common_pyutils.command_utils import run_command_with_output
+
+
+class NextFlowProcess:
+
+    def __init__(self, process_name: str, command_to_run: str, process_directives: Dict[str, str] = None) -> None:
+        """
+        Create a Nextflow process
+        :rtype: None
+        :param process_name: Name of the process - should be a valid identifier - ex: p1_merge
+        :type process_name: str
+        :param command_to_run: Command to be run - ex: bash -c "echo p1"
+        :type command_to_run: str
+        :param process_directives: Additional process directives - ex: {"memory": "4GB", "executor": "lsf"}
+        :type process_directives: dict
+        """
+        if not process_name.isidentifier():
+            raise ValueError(f"{process_name} is not a valid Nextflow process name")
+        self.process_name = process_name
+        self.success_flag = f"{self.process_name}_success"
+        self.command_to_run = command_to_run
+        self.process_directives = process_directives if process_directives else dict()
+
+
+class NextFlowPipeline(AppLogger):
+    def __init__(self, process_dependency_map: Dict[NextFlowProcess, List[NextFlowProcess]] = None) -> None:
+        """
+        Create a Nextflow pipeline with a process dependency map
+
+        :param process_dependency_map: Map of Nextflow processes and their corresponding dependencies
+        - ex: {p3 : [p2], p2: [p1]}  where p1, p2 and p3 are Nextflow processes that should be executed sequentially
+        """
+        # Modeling the dependency map as a DiGraph (Directed graph) is advantageous
+        # in ordering/combining flows and detecting cycles
+        self.process_dependency_map = nx.ordered.DiGraph()
+        if process_dependency_map:
+            self.add_dependencies(process_dependency_map)
+
+    def add_dependencies(self, process_dependency_map: Dict[NextFlowProcess, List[NextFlowProcess]]):
+        for process, dependencies in process_dependency_map.items():
+            if dependencies:
+                for dependency in dependencies:
+                    self.add_process_dependency(process, dependency)
+            else:
+                self.add_process_dependency(process, None)
+
+    def add_process_dependency(self, process: NextFlowProcess, dependency: Union[NextFlowProcess, None]):
+        if dependency:
+            self.process_dependency_map.add_edge(process, dependency)
+            if not nx.dag.is_directed_acyclic_graph(self.process_dependency_map):
+                raise ValueError(f"Cycles found in pipeline when adding process {process.process_name} "
+                                 f"and its dependency {dependency.process_name}")
+        else:
+            # If no dependency is specified, the process will just be a single node in the DAG
+            self.process_dependency_map.add_node(process)
+
+    def _write_to_pipeline_file(self, workflow_file_path: str):
+        with open(workflow_file_path, "a") as pipeline_file_handle:
+            pipeline_file_handle.write(self.__str__() + "\n")
+
+    def run_pipeline(self, workflow_file_path: str, nextflow_binary_path: str = 'nextflow',
+                     nextflow_config_path: str = None, working_dir: str = ".", resume: bool = False,
+                     other_args: dict = None):
+        # Remove pipeline file if it already exists
+        if os.path.exists(workflow_file_path):
+            os.remove(workflow_file_path)
+        self._write_to_pipeline_file(workflow_file_path)
+        workflow_command = f"cd {working_dir} && {nextflow_binary_path} run {workflow_file_path}"
+        workflow_command += f" -c {nextflow_config_path}" if nextflow_config_path else ""
+        workflow_command += f" -with-report {workflow_file_path}.report.html"
+        workflow_command += f" -with-dag {workflow_file_path}.dag.png"
+        workflow_command += " -resume" if resume else ""
+        workflow_command += " ".join([f" -{arg} {val}" for arg, val in other_args.items()]) if other_args else ""
+        run_command_with_output(f"Running pipeline {workflow_file_path}...", workflow_command)
+
+    @staticmethod
+    def join_pipelines(main_pipeline: 'NextFlowPipeline', dependent_pipeline: 'NextFlowPipeline',
+                       with_dependencies: bool = True) -> 'NextFlowPipeline':
+        """
+        Join two pipelines with or without dependencies
+
+        With Dependencies it returns a new pipeline where:
+            1) root processes are those of the main pipeline.
+            2) final processes are those of the dependent pipeline and
+            3) every root process of the dependent pipeline depends on the final processes of the main pipeline.
+        Without Dependencies it returns a new pipeline where:
+            1) the two pipeline are left independent
+            2) Only shared dependencies
+            3) every root process of the dependent pipeline depends on the final processes of the main pipeline.
+
+        """
+        joined_pipeline = NextFlowPipeline()
+        # Aggregate dependency maps of both pipelines
+        joined_pipeline.process_dependency_map = nx.compose(main_pipeline.process_dependency_map,
+                                                            dependent_pipeline.process_dependency_map)
+        if with_dependencies:
+            for final_process_in_main_pipeline in main_pipeline._get_final_processes():
+                for root_process_in_dependent_pipeline in dependent_pipeline._get_root_processes():
+                    joined_pipeline.add_process_dependency(root_process_in_dependent_pipeline,
+                                                           final_process_in_main_pipeline)
+        return joined_pipeline
+
+    def _get_root_processes(self) -> List[NextFlowProcess]:
+        # Root processes are those which have no dependencies
+        # See https://stackoverflow.com/a/62948641
+        roots = []
+        for component in nx.weakly_connected_components(self.process_dependency_map):
+            subgraph = self.process_dependency_map.subgraph(component)
+            roots.extend([n for n, d in subgraph.out_degree() if d == 0])
+        return roots
+
+    def _get_final_processes(self) -> List[NextFlowProcess]:
+        # Final processes are those which have no other processes depending on them
+        # See https://stackoverflow.com/a/62948641
+        roots = []
+        for component in nx.weakly_connected_components(self.process_dependency_map):
+            subgraph = self.process_dependency_map.subgraph(component)
+            roots.extend([n for n, d in subgraph.in_degree() if d == 0])
+        return roots
+
+    @staticmethod
+    def _get_process_repr(process: NextFlowProcess, dependencies: List[NextFlowProcess]) -> str:
+        process_directives_str = "\n".join([f"{key}='{value}'" for key, value in process.process_directives.items()])
+        input_dependencies = "val flag from true"
+        if dependencies:
+            input_dependencies = "\n".join([f"val {dependency.success_flag} from {dependency.success_flag}"
+                                            for dependency in dependencies])
+        return "\n".join(map(str.strip, f"""
+                    process {process.process_name} {{
+                    {process_directives_str}
+                    input:
+                    {input_dependencies}
+                    output:
+                    val true into {process.success_flag}
+                    script:
+                    \"\"\"
+                    {process.command_to_run}
+                    \"\"\"
+                    }}""".split("\n")))
+
+    def __str__(self):
+        # Order the list of nodes based on the dependency
+        # See https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.traversal.depth_first_search.dfs_postorder_nodes.html?highlight=dfs_postorder_nodes#networkx.algorithms.traversal.depth_first_search.dfs_postorder_nodes
+        ordered_list_of_processes_to_run = list(nx.dfs_postorder_nodes(self.process_dependency_map))
+        # Get a Nextflow pipeline representation of each process and its dependencies
+        return "\n\n".join([NextFlowPipeline._get_process_repr(process, list(self.process_dependency_map[process]))
+                            for process in ordered_list_of_processes_to_run])
+
+
+class LinearNextFlowPipeline(NextFlowPipeline):
+    """
+    Simple linear pipeline that supports resumption
+    """
+    previous_process: NextFlowProcess = None
+
+    def __init__(self, process_list: List[NextFlowProcess] = None):
+        dependency_map = {}
+        if process_list:
+            for index, process in enumerate(process_list):
+                dependency_map[process] = [] if index == 0 else [process_list[index - 1]]
+        super().__init__(dependency_map)
+
+    def add_process(self, process_name, command_to_run):
+        current_process = NextFlowProcess(process_name=process_name, command_to_run=command_to_run)
+        self._add_new_process(current_process)
+
+    def _add_new_process(self, current_process):
+        super().add_process_dependency(current_process, self.previous_process)
+        self.previous_process = current_process

ebi_eva_internal_pyutils/pg_utils.py

@@ -0,0 +1,107 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import psycopg2
+from ebi_eva_common_pyutils.logger import logging_config as log_cfg
+
+logger = log_cfg.get_logger(__name__)
+
+
+def get_all_results_for_query(pg_conn, query):
+    with get_result_cursor(pg_conn, query) as pg_cursor:
+        results = pg_cursor.fetchall()
+    return results
+
+
+def execute_query(pg_conn, query):
+    with get_result_cursor(pg_conn, query) as _:
+        pg_conn.commit()
+
+
+def get_result_cursor(pg_conn, query):
+    pg_cursor = pg_conn.cursor()
+    pg_cursor.execute(query)
+    return pg_cursor
+
+
+def get_pg_connection_handle(dbname, user, host):
+    return psycopg2.connect("dbname='{0}' user='{1}' host='{2}'".format(dbname, user, host))
+
+
+def index_already_exists_on_table(pg_conn, schema_name, table_name, index_columns):
+    index_columns_lower_case = list(map(str.lower, index_columns))
+    query = """select unnest(column_names) from (
+                    select
+                        nmsp.nspname as schema_name,
+                        t.relname as table_name,
+                        i.relname as index_name,
+                        array_agg(a.attname) as column_names,
+                        count(*) as number_of_columns
+                    from
+                        pg_class t,
+                        pg_class i,
+                        pg_index ix,
+                        pg_attribute a,
+                        pg_namespace nmsp
+                    where
+                        t.oid = ix.indrelid
+                        and i.oid = ix.indexrelid
+                        and a.attrelid = t.oid
+                        and a.attnum = ANY(ix.indkey)
+                        and t.relkind = 'r'
+                        and nmsp.oid = t.relnamespace
+                        and nmsp.nspname = '{0}'
+                        and t.relname = '{1}'
+                        and a.attname in ({2})                        
+                    group by    
+                        schema_name, table_name, index_name
+                    order by
+                        t.relname,
+                        i.relname
+                    ) temp 
+        where number_of_columns = {3};
+    """.format(schema_name, table_name,
+               ",".join(["'{0}'".format(col) for col in index_columns_lower_case]), len(index_columns))
+    results = [result[0] for result in get_all_results_for_query(pg_conn, query)]
+    return sorted(results) == index_columns_lower_case
+
+
+def create_index_on_table(pg_conn, schema_name, table_name, index_columns):
+    if index_already_exists_on_table(pg_conn, schema_name, table_name, index_columns):
+        logger.info("Index on {0} column(s) on {1}.{2} already exists. Skipping..."
+                    .format(",".join(list(map(str.lower, sorted(index_columns)))), schema_name, table_name))
+    else:
+        query = "create index on {0}.{1} ({2})".format(schema_name, table_name,
+                                                       ",".join(list(map(str.lower, sorted(index_columns))))
+                                                       )
+        logger.info("Building index with query: " + query)
+        execute_query(pg_conn, query)
+        pg_conn.commit()
+
+
+def vacuum_analyze_table(pg_conn, schema_name, table_name, columns=()):
+    query = "vacuum analyze {0}.{1}".format(schema_name, table_name)
+    if columns:
+        query += "({0})".format(",".join(columns))
+    isolation_level_pre_analyze = pg_conn.isolation_level
+    try:
+        # This is needed for vacuum analyze to work since it can't work inside transactions!
+        pg_conn.set_isolation_level(0)
+        logger.info("Vacuum analyze with query: " + query)
+        execute_query(pg_conn, query)
+    except Exception as ex:
+        logger.error(ex)
+    finally:
+        pg_conn.set_isolation_level(isolation_level_pre_analyze)

ebi_eva_internal_pyutils/spring_properties.py

@@ -0,0 +1,294 @@
+# Copyright 2022 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from collections import defaultdict
+from urllib.parse import quote_plus
+
+from ebi_eva_internal_pyutils.config_utils import get_mongo_creds_for_profile, get_accession_pg_creds_for_profile, \
+    get_count_service_creds_for_profile, get_properties_from_xml_file, get_variant_load_job_tracker_creds_for_profile
+
+
+class SpringPropertiesGenerator:
+    """
+    Class to generate Spring properties for various Spring Batch pipelines.
+    These methods can be used to generate complete properties files entirely in Python; alternatively, certain
+    properties can be left unfilled and supplied as command-line arguments (e.g. by a NextFlow process).
+    """
+
+    def __init__(self, maven_profile, private_settings_file):
+        self.maven_profile = maven_profile
+        self.private_settings_file = private_settings_file
+
+    @staticmethod
+    def _format(*key_value_maps):
+        all_params = defaultdict(list)
+        for key_value_map in key_value_maps:
+            for key in key_value_map:
+                if key_value_map[key] is not None:
+                    all_params[key.split('.')[0]].append(f'{key}={key_value_map[key]}')
+        lines = []
+        for key_type in all_params:
+            for line in all_params[key_type]:
+                lines.append(line)
+            lines.append('')
+
+        return '\n'.join(lines)
+
+    @staticmethod
+    def _format_str(string, param):
+        if param is None:
+            return None
+        elif not param:
+            return ''
+        else:
+            return string.format(param)
+
+    def _mongo_properties(self):
+        mongo_host, mongo_user, mongo_pass = get_mongo_creds_for_profile(
+            self.maven_profile, self.private_settings_file)
+        username_with_password = (f'{quote_plus(mongo_user)}:{quote_plus(mongo_pass)}@'
+                                  if mongo_user is not None and mongo_pass is not None else '')
+        return {
+            'spring.data.mongodb.uri': f'mongodb://{username_with_password}{mongo_host}/?retryWrites=true&authSource=admin',
+        }
+
+    def _variant_load_job_tracker_properties(self):
+        variant_url, variant_user, variant_pass = get_variant_load_job_tracker_creds_for_profile(self.maven_profile,
+                                                                                                 self.private_settings_file)
+        return {
+            'job.repository.url': variant_url,
+            'job.repository.username': variant_user,
+            'job.repository.password': variant_pass,
+        }
+
+    def _count_stats_properties(self):
+        counts_url, counts_username, counts_password = get_count_service_creds_for_profile(
+            self.maven_profile, self.private_settings_file)
+        return {
+            'eva.count-stats.url': counts_url,
+            'eva.count-stats.username': counts_username,
+            'eva.count-stats.password': counts_password
+        }
+
+    def _common_properties(self, *, read_preference='primary', chunk_size=100, max_pool_size=2):
+        """Properties common to all Spring pipelines"""
+        props = {
+            'spring.datasource.driver-class-name': 'org.postgresql.Driver',
+            'spring.datasource.tomcat.max-active': 3,
+            'spring.jpa.generate-ddl': 'true',
+
+            'mongodb.read-preference': read_preference,
+
+            'spring.main.web-application-type': 'none',
+            'spring.main.allow-bean-definition-overriding': 'true',
+            'spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation': 'true',
+            'spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults': 'false',
+            'spring.jpa.database-platform': 'org.hibernate.dialect.PostgreSQL9Dialect',
+            'parameters.chunkSize': chunk_size,
+            'spring.datasource.hikari.maximum-pool-size': max_pool_size
+        }
+        merge = {**self._mongo_properties(), **self._count_stats_properties(), **props}
+        return merge
+
+    def _common_accessioning_properties(self, assembly_accession, read_preference, chunk_size):
+        pg_url, pg_user, pg_pass = get_accession_pg_creds_for_profile(self.maven_profile, self.private_settings_file)
+        accession_db = get_properties_from_xml_file(
+            self.maven_profile, self.private_settings_file)['eva.accession.mongo.database']
+        props = {
+            'spring.datasource.url': pg_url,
+            'spring.datasource.username': pg_user,
+            'spring.datasource.password': pg_pass,
+            'spring.data.mongodb.database': accession_db,
+            'parameters.assemblyAccession': assembly_accession,
+        }
+
+        merge = {**self._common_properties(read_preference=read_preference, chunk_size=chunk_size), **props}
+        return merge
+
+    def _common_accessioning_clustering_properties(self, *, assembly_accession, read_preference, chunk_size):
+        """Properties common to accessioning and clustering pipelines."""
+        props = {
+            'accessioning.submitted.categoryId': 'ss',
+            'accessioning.clustered.categoryId': 'rs',
+            'accessioning.monotonic.ss.blockSize': 100000,
+            'accessioning.monotonic.ss.blockStartValue': 5000000000,
+            'accessioning.monotonic.ss.nextBlockInterval': 1000000000,
+            'accessioning.monotonic.rs.blockSize': 100000,
+            'accessioning.monotonic.rs.blockStartValue': 3000000000,
+            'accessioning.monotonic.rs.nextBlockInterval': 1000000000,
+            # This value is not used but is required to create beans in Java
+            'recovery.cutoff.days': 9999999
+        }
+        merge = {**self._common_accessioning_properties(assembly_accession, read_preference, chunk_size), **props}
+        return merge
+
+    def get_accessioning_properties(self, *, target_assembly=None, fasta=None, assembly_report=None,
+                                    project_accession=None, aggregation='BASIC', taxonomy_accession=None,
+                                    vcf_file='', output_vcf='', chunk_size=100):
+        """Properties for accessioning pipeline."""
+        return self._format(
+            self._common_accessioning_clustering_properties(assembly_accession=target_assembly,
+                                                            read_preference='secondaryPreferred',
+                                                            chunk_size=chunk_size),
+            {
+                'spring.batch.job.names': 'CREATE_SUBSNP_ACCESSION_JOB',
+                'parameters.assemblyReportUrl': self._format_str('file:{0}', assembly_report),
+                'parameters.contigNaming': 'NO_REPLACEMENT',
+                'parameters.fasta': fasta,
+                'parameters.forceRestart': 'false',
+                'parameters.projectAccession': project_accession,
+                'parameters.taxonomyAccession': taxonomy_accession,
+                'parameters.vcfAggregation': aggregation,
+                'parameters.vcf': vcf_file,
+                'parameters.outputVcf': output_vcf
+            },
+        )
+
+    def get_clustering_properties(self, *, read_preference='primary', job_name=None, source_assembly='',
+                                  target_assembly='', rs_report_path='', rs_acc_file='', duplicate_rs_acc_file='',
+                                  projects='', project_accession='', vcf=''):
+        """Properties common to all clustering pipelines, though not all are always used."""
+        return self._format(
+            self._common_accessioning_clustering_properties(assembly_accession=target_assembly,
+                                                            read_preference=read_preference, chunk_size=100),
+            {
+                'spring.batch.job.names': job_name,
+                'parameters.remappedFrom': source_assembly,
+                'parameters.projects': projects,
+                'parameters.projectAccession': project_accession,
+                'parameters.vcf': vcf,
+                'parameters.rsReportPath': rs_report_path,
+                'parameters.rsAccFile': rs_acc_file,
+                'parameters.duplicateRSAccFile': duplicate_rs_acc_file,
+            }
+        )
+
+    def get_remapping_extraction_properties(self, *, taxonomy=None, source_assembly=None, fasta=None,
+                                            assembly_report=None,
+                                            projects='', output_folder=None):
+        """Properties for remapping extraction pipeline."""
+        return self._format(
+            self._common_accessioning_properties(assembly_accession=source_assembly,
+                                                 read_preference='secondaryPreferred',
+                                                 chunk_size=1000),
+            {
+                'spring.batch.job.names': 'EXPORT_SUBMITTED_VARIANTS_JOB',
+                'parameters.taxonomy': taxonomy,
+                'parameters.fasta': fasta,
+                'parameters.assemblyReportUrl': self._format_str('file:{0}', assembly_report),
+                'parameters.projects': projects,
+                'parameters.outputFolder': output_folder
+            })
+
+    def get_remapping_ingestion_properties(self, *, source_assembly=None, target_assembly=None, vcf=None, load_to=None,
+                                           remapping_version=1.0):
+        """Properties for remapping ingestion pipeline."""
+        return self._format(
+            self._common_accessioning_properties(assembly_accession=target_assembly,
+                                                 read_preference='secondaryPreferred',
+                                                 chunk_size=1000),
+            {
+                'spring.batch.job.names': 'INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB',
+                'parameters.vcf': vcf,
+                'parameters.remappedFrom': source_assembly,
+                'parameters.loadTo': load_to,
+                'parameters.remappingVersion': remapping_version,
+            }
+        )
+
+    def get_release_properties(self, *, job_name=None, assembly_accession=None, taxonomy_accession=None, fasta=None,
+                               assembly_report=None, contig_naming=None, output_folder=None, accessioned_vcf=None,
+                               temp_mongo_db=None):
+        common_props = self._common_accessioning_properties(assembly_accession=assembly_accession,
+                                                            read_preference='secondaryPreferred', chunk_size=1000)
+        # For release in Embassy only
+        if temp_mongo_db:
+            common_props['spring.data.mongodb.database'] = temp_mongo_db
+            common_props['mongodb.read-preference'] = 'primaryPreferred'
+            common_props.pop('spring.data.mongodb.host')
+            common_props.pop('spring.data.mongodb.port')
+            common_props.pop('spring.data.mongodb.username')
+            common_props.pop('spring.data.mongodb.password')
+        return self._format(
+            common_props,
+            {
+                'spring.batch.job.names': job_name,
+                'parameters.taxonomyAccession': taxonomy_accession,
+                'parameters.contigNaming': contig_naming,
+                'parameters.fasta': fasta,
+                'parameters.assemblyReportUrl': self._format_str('file:{0}', assembly_report),
+                'parameters.outputFolder': output_folder,
+                'parameters.accessionedVcf': '' if accessioned_vcf is None else accessioned_vcf,
+                'logging.level.uk.ac.ebi.eva.accession.release': 'INFO'
+            })
+
+    def _common_eva_pipeline_properties(self, opencga_path, read_preference='secondaryPreferred'):
+        files_collection = get_properties_from_xml_file(
+            self.maven_profile, self.private_settings_file)['eva.mongo.collections.files']
+        annotation_metadata_collection = get_properties_from_xml_file(
+            self.maven_profile, self.private_settings_file)['eva.mongo.collections.annotation-metadata']
+        annotation_collection = get_properties_from_xml_file(
+            self.maven_profile, self.private_settings_file)['eva.mongo.collections.annotations']
+        variants_collection = get_properties_from_xml_file(
+            self.maven_profile, self.private_settings_file)['eva.mongo.collections.variants']
+        job_tracker_properties = self._variant_load_job_tracker_properties()
+        props = {
+            'spring.profiles.active': 'production,mongo',
+            'spring.profiles.include': 'variant-writer-mongo,variant-annotation-mongo',
+
+            'spring.data.mongodb.authentication-mechanism': 'SCRAM-SHA-1',
+            'job.repository.driverClassName': 'org.postgresql.Driver',
+
+            'db.collections.variants.name': variants_collection,
+            'db.collections.files.name': files_collection,
+            'db.collections.annotation-metadata.name': annotation_metadata_collection,
+            'db.collections.annotations.name': annotation_collection,
+
+            'app.opencga.path': opencga_path,
+            'config.restartability.allow': 'false',
+            'config.db.read-preference': read_preference,
+
+            'logging.level.embl.ebi.variation.eva': 'DEBUG',
+            'logging.level.org.opencb.opencga': 'DEBUG',
+            'logging.level.org.springframework': 'INFO',
+        }
+
+        merge = {**self._common_properties(read_preference=read_preference, chunk_size=100), **props,
+                 **job_tracker_properties}
+        return merge
+
+    def get_accession_import_properties(self, opencga_path, read_preference='secondaryPreferred'):
+        return self._format(self._common_eva_pipeline_properties(opencga_path, read_preference))
+
+    def get_variant_load_properties(self, project_accession, study_name, output_dir, annotation_dir, stats_dir,
+                                    vep_cache_path, opencga_path, read_preference='secondaryPreferred'):
+        return self._format(
+            self._common_eva_pipeline_properties(opencga_path, read_preference),
+            {
+                'annotation.overwrite': False,
+                'app.vep.cache.path': vep_cache_path,
+                'app.vep.num-forks': 4,
+                'app.vep.timeout': 500,
+                'config.chunk.size': 200,
+
+                'input.study.id': project_accession,
+                'input.study.name': study_name,
+                'input.study.type': 'COLLECTION',
+
+                'output.dir': str(output_dir),
+                'output.dir.annotation': str(annotation_dir),
+                'output.dir.statistics': str(stats_dir),
+
+                'statistics.skip': False
+            },
+        )