ebi-eva-common-pyutils 0.6.17__2-py3-none-any.whl

ebi_eva_common_pyutils-0.6.17.dist-info/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

ebi_eva_common_pyutils-0.6.17.dist-info/METADATA

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: ebi-eva-common-pyutils
+Version: 0.6.17
+Summary: EBI EVA - Common Python Utilities
+Home-page: https://github.com/EBIVariation/eva-common-pyutils
+License: Apache
+Keywords: EBI,EVA,PYTHON,UTILITIES
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+License-File: LICENSE
+Requires-Dist: requests (==2.*)
+Requires-Dist: lxml (==4.*,>4.9)
+Requires-Dist: pyyaml (==6.*)
+Requires-Dist: cached-property (==1.5.*)
+Requires-Dist: retry (==0.*,>0.9)
+Requires-Dist: openpyxl (==3.*)
+Provides-Extra: eva-internal
+Requires-Dist: psycopg2-binary ; extra == 'eva-internal'
+Requires-Dist: pymongo ; extra == 'eva-internal'
+Requires-Dist: networkx (<=2.5) ; extra == 'eva-internal'
+

ebi_eva_common_pyutils-0.6.17.dist-info/RECORD

@@ -0,0 +1,41 @@
+ebi_eva_common_pyutils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ebi_eva_common_pyutils/assembly_utils.py,sha256=CklyCGlCjlFp0e9pugg6kSsh5L0xfCe2qPvA2eLVtn0,4187
+ebi_eva_common_pyutils/biosamples_communicators.py,sha256=nyO0E_YpcAeBwbqOfbqe9RDxONq7Oe3NijSTGw7D01I,7815
+ebi_eva_common_pyutils/command_utils.py,sha256=PtelWWqcC0eOwIVesjwBw3F9KaXRzEE_uAUJhQFZ4l8,2340
+ebi_eva_common_pyutils/common_utils.py,sha256=ty_glvfRa3VGhnpAht4qtVkNNmv-IYfVtO958mY-BaA,1192
+ebi_eva_common_pyutils/config.py,sha256=PtD2SgHf96kk21OA9tVIjEgsDXEFuAU-INy_kfQdoPw,4828
+ebi_eva_common_pyutils/ena_utils.py,sha256=S2MmnWQ_9MJjlkaQY_by1-GGbTyi8SKp8XRcpjWnpZs,1465
+ebi_eva_common_pyutils/file_utils.py,sha256=eIlQKSVKkEjMNX7emrDzaQyQdGvQdb64gnfEhb6uYsE,1375
+ebi_eva_common_pyutils/logger.py,sha256=hT20ktN_oGeB_5ofVfd_aGXB6xYOe0Y5b3EVRYCuFb8,5093
+ebi_eva_common_pyutils/ncbi_utils.py,sha256=AKuNKv4ue7l-6rwuDDGubWEHGuscXWfv-Gg6o_x1hT0,5005
+ebi_eva_common_pyutils/network_utils.py,sha256=iJjs5PPzT1V4CceZnCHOTs711AmpwlDo59wb1XHdTak,2648
+ebi_eva_common_pyutils/assembly/__init__.py,sha256=KSWPwBY5nZj00odxWFntk8Sqg_rw273xH8S5D6Jo-T4,67
+ebi_eva_common_pyutils/assembly/assembly.py,sha256=IEmleROX4ZchPyhINKCuMmET_Ih1Jg4ok-opAKY6Z9A,3142
+ebi_eva_common_pyutils/contig_alias/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ebi_eva_common_pyutils/contig_alias/contig_alias.py,sha256=7tQXljEJtRCMs_JPrglYvTEZ-fJvzl8txwnCvqRB3Ck,5214
+ebi_eva_common_pyutils/reference/__init__.py,sha256=NsDjGCu2H2CZLOPaEL6GCoZo5ND_EA9RFmbzGfpSXRQ,134
+ebi_eva_common_pyutils/reference/assembly.py,sha256=FK4zsvg4_GDeKd9z2z5ZeM9R84TsusgQYdMOGuggQU0,12162
+ebi_eva_common_pyutils/reference/sequence.py,sha256=bg96QcuB-oytQYpaUkV10OzM_RIFhxsy6t3PQOg2Gy0,3911
+ebi_eva_common_pyutils/spreadsheet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ebi_eva_common_pyutils/spreadsheet/metadata_xlsx_utils.py,sha256=0yOoV5rfLc5IWPAShSClaAOmaGfJuGZjJOnREfZBtFQ,540
+ebi_eva_common_pyutils/taxonomy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ebi_eva_common_pyutils/taxonomy/taxonomy.py,sha256=aXmRQ3NAaJotwmmOA2-u2XtcUT6iih-0_e-3QOxynoA,2578
+ebi_eva_common_pyutils/variation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ebi_eva_common_pyutils/variation/contig_utils.py,sha256=kMNEW_P2yPnd8Xx1tep19hy5ee7ojxz6ZOO1grTQsRQ,5230
+ebi_eva_common_pyutils-0.6.17.data/scripts/archive_directory.py,sha256=0lWJ0ju_AB2ni7lMnJXPFx6U2OdTGbe-WoQs-4BfKOM,4976
+ebi_eva_internal_pyutils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ebi_eva_internal_pyutils/archive_directory.py,sha256=IxVEfh_gaCiT652k0Q_-58fonRusy1yzXu7BCO8yVLo,4989
+ebi_eva_internal_pyutils/config_utils.py,sha256=EGRC5rsmU_ug7OY9-t1UW1XZXRsauSyZB9xPcBux8ts,7909
+ebi_eva_internal_pyutils/metadata_utils.py,sha256=t9PcXZdbfjDBP04GJenC4bxm2nOLd8oI_MP9eNe9IBQ,15221
+ebi_eva_internal_pyutils/mongo_utils.py,sha256=YxKHtb5ygDiGLOtEiiAMFCP2ow6FL9Kq0K5R0mWNdXY,3575
+ebi_eva_internal_pyutils/pg_utils.py,sha256=FUQVwiX_7F2-4sSzoaCVX2me0zAqR8nGIj6NW5d304A,4398
+ebi_eva_internal_pyutils/spring_properties.py,sha256=Tn207DmZehFt7oExseNsXFAnsxr7bX9yiGl4t9mpGVA,15165
+ebi_eva_internal_pyutils/mongodb/__init__.py,sha256=0oyTlkYZCV7udlPl09Zl-sDyE3c97QZMMTEFIa6uYIw,76
+ebi_eva_internal_pyutils/mongodb/mongo_database.py,sha256=kesaJaaxYFeF_uYZBgL8tbufGKUXll7bXb4WlOS9vKM,9596
+ebi_eva_internal_pyutils/nextflow/__init__.py,sha256=OOiJS8jZOz98q0t77NNog7aI_fFrVxi4kGmiSskuAqM,122
+ebi_eva_internal_pyutils/nextflow/nextflow_pipeline.py,sha256=ew623hhK8jmFLQjJwLZbgBmW9RTiJBEULVqHfIUv_dc,10114
+ebi_eva_common_pyutils-0.6.17.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ebi_eva_common_pyutils-0.6.17.dist-info/METADATA,sha256=xdXBYa6ma2iy0NDeoYjrral5mPQnsC-F63kQetBZx5Q,909
+ebi_eva_common_pyutils-0.6.17.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+ebi_eva_common_pyutils-0.6.17.dist-info/top_level.txt,sha256=sXoiqiGU8vlMQpFWDlKrekxhlusk06AhkOH3kSvDT6c,48
+ebi_eva_common_pyutils-0.6.17.dist-info/RECORD,,

ebi_eva_common_pyutils-0.6.17.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.38.4)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ebi_eva_common_pyutils-0.6.17.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+ebi_eva_common_pyutils
+ebi_eva_internal_pyutils

ebi_eva_internal_pyutils/archive_directory.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+import gzip
+import shutil
+import tarfile
+import os.path
+from argparse import ArgumentParser
+
+from ebi_eva_common_pyutils.logger import logging_config
+from retry import retry
+
+logger = logging_config.get_logger(__name__)
+
+
+def make_tarfile(output_filename, source_dir):
+    logger.info(f'Create Final Tar file {output_filename}.')
+    with tarfile.open(output_filename, "w") as tar:
+        tar.add(source_dir, arcname=os.path.basename(source_dir))
+    file_stats = os.stat(output_filename)
+    logger.info(f'{output_filename} completed. File Size in Bytes is {file_stats.st_size}')
+
+
+def is_compressed(file_path):
+    compressed_ext = ['.gz', '.zip', '.bz', '.tbi', '.csi']
+    for ext in compressed_ext:
+        if file_path.lower().endswith(ext):
+            return True
+    return False
+
+
+@retry(tries=5, delay=3, backoff=2, logger=logger)
+def retriable_compress(src_file_path, dest_file_path):
+    MEG = 2 ** 20
+    with open(src_file_path, 'rb') as f_in:
+        with gzip.open(dest_file_path, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out, length=16 * MEG)
+
+
+def matches(name, patterns):
+    return any((pattern for pattern in patterns if pattern in name))
+
+
+@retry(tries=5, delay=3, backoff=2, logger=logger)
+def retriable_remove(path):
+    shutil.rmtree(path)
+
+
+@retry(tries=5, delay=3, backoff=2, logger=logger)
+def retryable_copy(src_file_path, dest_file_path, **kwargs):
+    if os.path.exists(dest_file_path):
+        os.remove(dest_file_path)
+    shutil.copyfile(src_file_path, dest_file_path, **kwargs)
+
+
+def archive_directory(source_dir, scratch_dir, destination_dir, filter_patterns=None):
+    """
+    Archive a directory by copying the data it contains to a scratch directory compressing all files that are not
+    already compressed. Then it create a tar file to the destination_dir under the name of the original directory.
+    """
+    source_dir_name = os.path.basename(source_dir)
+    logger.info(f'Archive {source_dir_name} from {source_dir}')
+    parent_source_dir = os.path.dirname(source_dir)
+    for base, dirs, files in os.walk(source_dir, topdown=True, followlinks=False):
+        # Filter the downstream directory to
+        filtered_dir = []
+        for d in dirs:
+            if matches(d, filter_patterns):
+                logger.info(f'Ignore directory {d} because of filters: {filter_patterns}')
+            else:
+                filtered_dir.append(d)
+        # modify dirs in place
+        dirs[:] = filtered_dir
+        src_basename = os.path.relpath(base, parent_source_dir)
+        scratch_dest_dir = os.path.join(scratch_dir, src_basename)
+        os.makedirs(scratch_dest_dir, exist_ok=True)
+        for fname in files:
+            src_file_path = os.path.join(base, fname)
+            dest_file_path = os.path.join(scratch_dest_dir, fname)
+            if matches(fname, filter_patterns):
+                logger.info(f'Ignore file {src_file_path} because of filters: {filter_patterns}')
+                continue
+            if os.path.islink(src_file_path) or is_compressed(src_file_path):
+                logger.info(f'Copy {src_file_path}')
+                retryable_copy(src_file_path, dest_file_path, follow_symlinks=False)
+            else:
+                logger.info(f'Compress {src_file_path}')
+                retriable_compress(src_file_path, dest_file_path + '.gz')
+    final_tar_file = os.path.join(destination_dir, source_dir_name + '.tar')
+    scratch_dir_archived = os.path.join(scratch_dir, source_dir_name)
+    make_tarfile(final_tar_file, scratch_dir_archived)
+    logger.info(f'Delete scratch folder {scratch_dir_archived}.')
+    retriable_remove(scratch_dir_archived)
+    logger.info(f'Scratch folder {scratch_dir_archived} deleted.')
+
+
+def main():
+    parser = ArgumentParser(description='Archive a directory by copying the data it contains to a scratch directory '
+                                        'compressing all files that are not already compressed. Then it create a tar '
+                                        'file to the destination_dir under the name of the original directory.')
+    parser.add_argument('--source_dir', required=True, type=str,
+                        help='base directory you want to archive. All sub directories will be included.')
+    parser.add_argument('--destination_dir', required=True, type=str,
+                        help='Directory where the archive should be placed at the end of the process.')
+    parser.add_argument('--scratch_dir', required=True, type=str,
+                        help='Directory where the archive will be constructed.')
+    parser.add_argument('--filter_patterns', type=str, nargs='*', default=[],
+                        help='keyword found in file and directory names that not be included to the archive.')
+    args = parser.parse_args()
+
+    logging_config.add_stdout_handler()
+    archive_directory(args.source_dir, args.scratch_dir,  args.destination_dir, args.filter_patterns)
+
+
+if __name__ == '__main__':
+    main()

ebi_eva_internal_pyutils/config_utils.py

@@ -0,0 +1,188 @@
+# Copyright 2020 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from urllib.parse import quote_plus
+from lxml import etree as et
+import json
+import yaml
+import urllib.request
+
+from retry import retry
+
+
+class EVAPrivateSettingsXMLConfig:
+    config_data = None
+
+    def __init__(self, settings_xml_file: str):
+        with open(settings_xml_file) as xml_file_handle:
+            self.config_data = et.parse(xml_file_handle)
+
+    def get_value_with_xpath(self, location: str, optional: bool = False):
+        etree = self.config_data.getroot()
+        result = etree.xpath(location)
+        if not result and not optional:
+            raise ValueError("Invalid XPath location: " + location)
+        return result
+
+
+def get_metadata_creds_for_profile(profile_name: str, settings_xml_file: str):
+    """
+    Gets host, username, and password for metadata postgres database.
+    Useful for filling properties files, for connection purposes you can use
+    `metadata_utils.get_metadata_connection_handle`.
+    """
+    properties = get_properties_from_xml_file(profile_name, settings_xml_file)
+    pg_url = properties['eva.evapro.jdbc.url']
+    pg_user = properties['eva.evapro.user']
+    pg_pass = properties['eva.evapro.password']
+    return pg_url, pg_user, pg_pass
+
+
+def get_mongo_creds_for_profile(profile_name: str, settings_xml_file: str):
+    """
+    Gets host, username, and password for mongo database.
+    Useful for filling properties files, for connection purposes it is preferable to use
+    `mongo_utils.get_mongo_connection_handle` as that will handle multiple hosts appropriately.
+    """
+    properties = get_properties_from_xml_file(profile_name, settings_xml_file)
+    mongo_host = properties['eva.mongo.host']
+    mongo_user = properties['eva.mongo.user']
+    mongo_pass = properties['eva.mongo.passwd']
+    return mongo_host, mongo_user, mongo_pass
+
+
+def get_accession_pg_creds_for_profile(profile_name: str, settings_xml_file: str):
+    """
+    Gets host, username, and password for accessioning job tracker database.
+    Useful for filling properties files.
+    """
+    properties = get_properties_from_xml_file(profile_name, settings_xml_file)
+    pg_url = properties['eva.accession.jdbc.url']
+    pg_user = properties['eva.accession.user']
+    pg_pass = properties['eva.accession.password']
+    return pg_url, pg_user, pg_pass
+
+
+def get_variant_load_job_tracker_creds_for_profile(profile_name: str, settings_xml_file: str):
+    """
+    Gets host, username, and password for variant load job tracker database.
+    Useful for filling properties files.
+    """
+    properties = get_properties_from_xml_file(profile_name, settings_xml_file)
+    variant_url = properties['eva.variant.jdbc.url']
+    variant_user = properties['eva.variant.user']
+    variant_pass = properties['eva.variant.password']
+    return variant_url, variant_user, variant_pass
+
+
+def get_contig_alias_db_creds_for_profile(profile_name: str, settings_xml_file: str):
+    """
+    Gets url, username, and password for contig alias database.
+
+    """
+    properties = get_properties_from_xml_file(profile_name, settings_xml_file)
+    contig_alias_url = properties['contig-alias.url']
+    contig_alias_user = properties['contig-alias.admin-user']
+    contig_alias_pass = properties['contig-alias.admin-password']
+
+    return contig_alias_url, contig_alias_user, contig_alias_pass
+
+
+def get_count_service_creds_for_profile(profile_name: str, settings_xml_file: str):
+    """
+    Gets host, username, and password for eva count service.
+    Useful for filling properties files.
+    """
+    properties = get_properties_from_xml_file(profile_name, settings_xml_file)
+    counts_url = properties['eva.count-stats.url']
+    counts_user = properties['eva.count-stats.username']
+    counts_pass = properties['eva.count-stats.password']
+    return counts_url, counts_user, counts_pass
+
+
+def get_pg_uri_for_accession_profile(profile_name: str, settings_xml_file: str):
+    return get_pg_uri_details_for_profile(profile_name, settings_xml_file, "eva.accession.jdbc.url")
+
+
+def get_pg_uri_for_variant_profile(profile_name: str, settings_xml_file: str):
+    return get_pg_uri_details_for_profile(profile_name, settings_xml_file, "eva.variant.jdbc.url")
+
+
+def get_pg_metadata_uri_for_eva_profile(profile_name: str, settings_xml_file: str):
+    return get_pg_uri_details_for_profile(profile_name, settings_xml_file, "eva.evapro.jdbc.url")
+
+
+def get_pg_uri_details_for_profile(eva_profile_name: str, settings_xml_file: str, tag_name: str):
+    config = EVAPrivateSettingsXMLConfig(settings_xml_file)
+    xpath_location_template = '//settings/profiles/profile/id[text()="{0}"]/../properties/{1}/text()'
+    # Format is jdbc:postgresql://host:port/db
+    metadata_db_jdbc_url = config.get_value_with_xpath(
+        xpath_location_template.format(eva_profile_name, tag_name))[0]
+    return metadata_db_jdbc_url.split("jdbc:")[-1]
+
+
+def get_mongo_uri_for_eva_profile(eva_profile_name: str, settings_xml_file: str):
+    config = EVAPrivateSettingsXMLConfig(settings_xml_file)
+    xpath_location_template = '//settings/profiles/profile/id[text()="{0}"]/../properties/{1}/text()'
+    # Format is host1:port1,host2:port2
+    mongo_hosts_and_ports = config.get_value_with_xpath(
+        xpath_location_template.format(eva_profile_name, "eva.mongo.host"))[0]
+    username = config.get_value_with_xpath(
+        xpath_location_template.format(eva_profile_name, "eva.mongo.user"), optional=True)
+    if not username:  # no authentication
+        return f"mongodb://{mongo_hosts_and_ports}"
+    username = username[0]
+    password = config.get_value_with_xpath(
+        xpath_location_template.format(eva_profile_name, "eva.mongo.passwd"))[0]
+    authentication_db = config.get_value_with_xpath(
+        xpath_location_template.format(eva_profile_name, "eva.mongo.auth.db"))[0]
+    return "mongodb://{0}:{1}@{2}/{3}".format(username, quote_plus(password), mongo_hosts_and_ports, authentication_db)
+
+
+def get_properties_from_xml_file(profile, xml_path):
+    tree = et.parse(xml_path)
+    root = tree.getroot()
+    return get_profile_properties(profile, root)
+
+
+def get_properties_from_xml_string(profile, str):
+    root = et.fromstring(str)
+    return get_profile_properties(profile, root)
+
+
+def get_profile_properties(profile, root):
+    properties = {}
+    for property in root.xpath('//settings/profiles/profile/id[text()="' + profile + '"]/../properties/*'):
+        properties[property.tag] = property.text
+    return properties
+
+
+@retry(tries=4, delay=2, backoff=1.2, jitter=(1, 3))
+def get_eva_settings_xml_string(token):
+    url = 'https://api.github.com/repos/EBIvariation/configuration/contents/eva-maven-settings.xml'
+    headers = {'Authorization': 'token ' + token, 'Accept' : 'application/vnd.github.raw' }
+    request = urllib.request.Request(url, None, headers)
+    with urllib.request.urlopen(request) as response:
+        return response.read()
+
+
+def get_args_from_private_config_file(private_config_file):
+    with open(private_config_file) as private_config_file_handle:
+        if 'json' in private_config_file:
+            return json.load(private_config_file_handle)
+        else:
+            if 'yml' in private_config_file:
+                return yaml.safe_load(private_config_file_handle)
+            else:
+                raise TypeError('Configuration file should be either json or yaml')