ebi-eva-common-pyutils 0.6.0__py3-none-any.whl → 0.6.4__py3-none-any.whl

ebi_eva_common_pyutils/assembly_utils.py

@@ -16,6 +16,7 @@ import http
 import requests
 
 from ebi_eva_common_pyutils.assembly import NCBIAssembly
+from ebi_eva_common_pyutils.ena_utils import download_xml_from_ena
 from ebi_eva_common_pyutils.logger import logging_config as log_cfg
 
 EUTILS_URL = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
@@ -27,6 +28,18 @@ EFETCH_URL = EUTILS_URL + 'efetch.fcgi'
 logger = log_cfg.get_logger(__name__)
 
 
+def is_patch_assembly(assembly_accession: str) -> bool:
+    """
+    Check if a given assembly is a patch assembly
+    Please see: https://www.ncbi.nlm.nih.gov/grc/help/patches/
+    """
+    xml_root = download_xml_from_ena(f'https://www.ebi.ac.uk/ena/browser/api/xml/{assembly_accession}')
+    xml_assembly = xml_root.xpath("//ASSEMBLY_ATTRIBUTE[TAG='count-patches']/VALUE")
+    if len(xml_assembly) == 0:
+        return False
+    return int(xml_assembly[0].text) > 0
+
+
 def retrieve_genbank_assembly_accessions_from_ncbi(assembly_txt):
     """
     Attempt to find any assembly genebank accession base on a free text search.

ebi_eva_common_pyutils/command_utils.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import logging
 import subprocess
 
 from ebi_eva_common_pyutils.logger import logging_config as log_cfg
@@ -20,11 +20,12 @@ logger = log_cfg.get_logger(__name__)
 
 
 def run_command_with_output(command_description, command, return_process_output=False,
-                            log_error_stream_to_output=False):
+                            log_error_stream_to_output=False, stdout_log_level=logging.INFO,
+                            stderr_log_level=logging.ERROR):
     process_output = ""
 
-    logger.info("Starting process: " + command_description)
-    logger.info("Running command: " + command)
+    logger.log(stdout_log_level, "Starting process: " + command_description)
+    logger.log(stdout_log_level, "Running command: " + command)
 
     stdout = subprocess.PIPE
     # Some lame utilities like mongodump and mongorestore output non-error messages to error stream
@@ -35,18 +36,18 @@ def run_command_with_output(command_description, command, return_process_output=
                           shell=True) as process:
         for line in iter(process.stdout.readline, ''):
             line = str(line).rstrip()
-            logger.info(line)
+            logger.log(stdout_log_level, line)
             if return_process_output:
                 process_output += line + "\n"
         if not log_error_stream_to_output:
             for line in iter(process.stderr.readline, ''):
                 line = str(line).rstrip()
-                logger.error(line)
+                logger.log(stderr_log_level, line)
     if process.returncode != 0:
         logger.error(command_description + " failed! Refer to the error messages for details.")
         raise subprocess.CalledProcessError(process.returncode, process.args)
     else:
-        logger.info(command_description + " - completed successfully")
+        logger.log(stdout_log_level, command_description + " - completed successfully")
     if return_process_output:
         return process_output
 

ebi_eva_common_pyutils/config.py

@@ -73,3 +73,80 @@ cfg = Configuration()
 """
 Provides a singleton that can be used as a central place for configuration.
 """
+
+
+
+
+class WritableConfig(Configuration):
+    """Configuration object that allows writes to the config file"""
+
+    def __init__(self, *search_path, version=None):
+        super().__init__(*search_path)
+        self.version = version
+
+    def load_config_file(self, *search_path):
+        try:
+            super().load_config_file(*search_path)
+        except FileNotFoundError:
+            # expected if it's the first time we are creating the config file
+            # In that case the first search path is set to be the config files
+            self.config_file = search_path[0]
+            pass
+
+    def backup(self):
+        """
+        Rename the config file by adding a '.1' at the end. If the '.1' file exists it move it to a '.2' and so on.
+        """
+        if os.path.isfile(self.config_file):
+            file_name = self.config_file
+            suffix = 1
+            backup_name = f'{file_name}.{suffix}'
+            while os.path.exists(backup_name):
+                suffix += 1
+                backup_name = f'{file_name}.{suffix}'
+
+            for i in range(suffix, 1, -1):
+                os.rename(f'{file_name}.{i - 1}', f'{file_name}.{i}')
+            os.rename(file_name, file_name + '.1')
+
+    def write(self):
+        if self.config_file and self.content and os.path.isdir(os.path.dirname(self.config_file)):
+            with open(self.config_file, 'w') as open_config:
+                yaml.safe_dump(self.content, open_config)
+
+    def set(self, *path, value):
+        self._set_version()
+        top_level = self.content
+        for p in path[:-1]:
+            if p not in top_level:
+                top_level[p] = {}
+            top_level = top_level[p]
+        top_level[path[-1]] = value
+
+    def pop(self, *path, default=None):
+        """Recursive dictionary pop with default"""
+        top_level = self.content
+        for p in path[:-1]:
+            if p not in top_level:
+                return default
+            top_level = top_level[p]
+        return top_level.pop(path[-1], default)
+
+    def is_empty(self):
+        return not self.content
+
+    def clear(self):
+        self.content = {}
+
+    def _set_version(self):
+        # If we're starting to fill in an empty config, set the version if available
+        if self.is_empty() and self.version:
+            self.content['version'] = self.version
+
+    def __contains__(self, item):
+        return item in self.content
+
+    def __setitem__(self, item, value):
+        """Allow dict-style write access, e.g. config['this']='that'."""
+        self._set_version()
+        self.content[item] = value

ebi_eva_common_pyutils/ena_utils.py

@@ -4,7 +4,7 @@ from retry import retry
 
 
 @retry(tries=3, delay=2, backoff=1.2, jitter=(1, 3))
-def download_xml_from_ena(ena_url):
+def download_xml_from_ena(ena_url) -> etree.XML:
     """Download and parse XML from ENA"""
     try:  # catches any kind of request error, including non-20X status code
         response = requests.get(ena_url)

ebi_eva_common_pyutils/logger.py

@@ -127,6 +127,9 @@ class AppLogger:
     """
     log_cfg = logging_config
 
+    def log(self, level, msg, *args, **kwargs):
+        self._logger.log(level, msg, *args, **kwargs)
+
     def debug(self, msg, *args):
         self._logger.debug(msg, *args)
 

{ebi_eva_common_pyutils-0.6.0.dist-info → ebi_eva_common_pyutils-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ebi-eva-common-pyutils
-Version: 0.6.0
+Version: 0.6.4
 Summary: EBI EVA - Common Python Utilities
 Home-page: https://github.com/EBIVariation/eva-common-pyutils
 License: Apache

{ebi_eva_common_pyutils-0.6.0.dist-info → ebi_eva_common_pyutils-0.6.4.dist-info}/RECORD

@@ -1,11 +1,11 @@
 ebi_eva_common_pyutils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ebi_eva_common_pyutils/assembly_utils.py,sha256=idvmNDKJHrzMhl0DZjmozXlWNVQWEW1BSslhFHxXatI,3482
-ebi_eva_common_pyutils/command_utils.py,sha256=PhXSS1LPSGaExkwwYJqjpG0agRB2dbOehbm1aAPNSqg,2151
+ebi_eva_common_pyutils/assembly_utils.py,sha256=Lc1bDf0xmi-AF6eyM7pWMgsjnxZmpVHvF3E5aEiKBEI,4018
+ebi_eva_common_pyutils/command_utils.py,sha256=PtelWWqcC0eOwIVesjwBw3F9KaXRzEE_uAUJhQFZ4l8,2340
 ebi_eva_common_pyutils/common_utils.py,sha256=ty_glvfRa3VGhnpAht4qtVkNNmv-IYfVtO958mY-BaA,1192
-ebi_eva_common_pyutils/config.py,sha256=5argukdpNFw0MlSSNws-IFEH8QnKXJzm3nnRtuN4hLE,2242
-ebi_eva_common_pyutils/ena_utils.py,sha256=3XBBLwQi00AZGzW0-x2qNi1KFaAPDxsr5GDmiBg9gbc,1452
+ebi_eva_common_pyutils/config.py,sha256=PtD2SgHf96kk21OA9tVIjEgsDXEFuAU-INy_kfQdoPw,4828
+ebi_eva_common_pyutils/ena_utils.py,sha256=S2MmnWQ_9MJjlkaQY_by1-GGbTyi8SKp8XRcpjWnpZs,1465
 ebi_eva_common_pyutils/file_utils.py,sha256=eIlQKSVKkEjMNX7emrDzaQyQdGvQdb64gnfEhb6uYsE,1375
-ebi_eva_common_pyutils/logger.py,sha256=KjnxquAZxJWlD2YvploiHD11JYzWZwdQidHx7Fy6wI4,4990
+ebi_eva_common_pyutils/logger.py,sha256=hT20ktN_oGeB_5ofVfd_aGXB6xYOe0Y5b3EVRYCuFb8,5093
 ebi_eva_common_pyutils/ncbi_utils.py,sha256=sh9E_oDhDtliihJ0-FZuDd7paeidW5GfqRr5vBTo2k4,4859
 ebi_eva_common_pyutils/network_utils.py,sha256=_Qf5oNONA4pUzvxfdk25G-dKuWe44aC_Hewvdp-VZyU,2285
 ebi_eva_common_pyutils/assembly/__init__.py,sha256=KSWPwBY5nZj00odxWFntk8Sqg_rw273xH8S5D6Jo-T4,67
@@ -19,20 +19,20 @@ ebi_eva_common_pyutils/taxonomy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 ebi_eva_common_pyutils/taxonomy/taxonomy.py,sha256=p3XV4g3y0hEjyeZ4PwgN7Q3Et9G515ctQkSIo1kdDbU,2259
 ebi_eva_common_pyutils/variation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ebi_eva_common_pyutils/variation/contig_utils.py,sha256=kMNEW_P2yPnd8Xx1tep19hy5ee7ojxz6ZOO1grTQsRQ,5230
-ebi_eva_common_pyutils-0.6.0.data/scripts/archive_directory.py,sha256=0lWJ0ju_AB2ni7lMnJXPFx6U2OdTGbe-WoQs-4BfKOM,4976
+ebi_eva_common_pyutils-0.6.4.data/scripts/archive_directory.py,sha256=0lWJ0ju_AB2ni7lMnJXPFx6U2OdTGbe-WoQs-4BfKOM,4976
 ebi_eva_internal_pyutils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ebi_eva_internal_pyutils/archive_directory.py,sha256=IxVEfh_gaCiT652k0Q_-58fonRusy1yzXu7BCO8yVLo,4989
 ebi_eva_internal_pyutils/config_utils.py,sha256=EGRC5rsmU_ug7OY9-t1UW1XZXRsauSyZB9xPcBux8ts,7909
-ebi_eva_internal_pyutils/metadata_utils.py,sha256=sW7mzFooeXcmxMth5EiBhES-7Keh6Ooe4XeCubLWfJ4,14795
+ebi_eva_internal_pyutils/metadata_utils.py,sha256=NQVD0_-S1EBb0SnlKX0w31g4Me_8VGXYJTDlMhzje9w,15090
 ebi_eva_internal_pyutils/mongo_utils.py,sha256=YxKHtb5ygDiGLOtEiiAMFCP2ow6FL9Kq0K5R0mWNdXY,3575
 ebi_eva_internal_pyutils/pg_utils.py,sha256=FUQVwiX_7F2-4sSzoaCVX2me0zAqR8nGIj6NW5d304A,4398
 ebi_eva_internal_pyutils/spring_properties.py,sha256=WjPozWtXbAZGNqlgvY6GHps2KFB1rY9OaTs46obW3pM,15265
 ebi_eva_internal_pyutils/mongodb/__init__.py,sha256=0oyTlkYZCV7udlPl09Zl-sDyE3c97QZMMTEFIa6uYIw,76
-ebi_eva_internal_pyutils/mongodb/mongo_database.py,sha256=im0HgBgG_9E1nq7fQcNl3Ogc8oka6DI97SgsfeboRm0,9676
+ebi_eva_internal_pyutils/mongodb/mongo_database.py,sha256=kesaJaaxYFeF_uYZBgL8tbufGKUXll7bXb4WlOS9vKM,9596
 ebi_eva_internal_pyutils/nextflow/__init__.py,sha256=OOiJS8jZOz98q0t77NNog7aI_fFrVxi4kGmiSskuAqM,122
 ebi_eva_internal_pyutils/nextflow/nextflow_pipeline.py,sha256=ew623hhK8jmFLQjJwLZbgBmW9RTiJBEULVqHfIUv_dc,10114
-ebi_eva_common_pyutils-0.6.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ebi_eva_common_pyutils-0.6.0.dist-info/METADATA,sha256=cW4V0_SM_bvGOMTS6hHp7FmCSr6EVBZkLoHLSPPAXRc,824
-ebi_eva_common_pyutils-0.6.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-ebi_eva_common_pyutils-0.6.0.dist-info/top_level.txt,sha256=sXoiqiGU8vlMQpFWDlKrekxhlusk06AhkOH3kSvDT6c,48
-ebi_eva_common_pyutils-0.6.0.dist-info/RECORD,,
+ebi_eva_common_pyutils-0.6.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ebi_eva_common_pyutils-0.6.4.dist-info/METADATA,sha256=z3BrpkHtxmrrusChh4Oawx8m6tAeRLZpXLlN8pnu3ic,824
+ebi_eva_common_pyutils-0.6.4.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+ebi_eva_common_pyutils-0.6.4.dist-info/top_level.txt,sha256=sXoiqiGU8vlMQpFWDlKrekxhlusk06AhkOH3kSvDT6c,48
+ebi_eva_common_pyutils-0.6.4.dist-info/RECORD,,

ebi_eva_internal_pyutils/metadata_utils.py

@@ -17,6 +17,7 @@ from urllib.parse import urlsplit
 
 import psycopg2
 
+from ebi_eva_common_pyutils.assembly_utils import is_patch_assembly
 from ebi_eva_internal_pyutils.config_utils import get_metadata_creds_for_profile
 from ebi_eva_common_pyutils.ena_utils import get_scientific_name_and_common_name
 from ebi_eva_common_pyutils.logger import logging_config
@@ -120,6 +121,9 @@ def get_assembly_code(metadata_connection_handle, assembly):
     assembly_code = get_assembly_code_from_metadata(metadata_connection_handle, assembly)
     if not assembly_code:
         assembly_name = get_ncbi_assembly_name_from_term(assembly)
+        # If the assembly is a patch assembly ex: GRCh37.p8, drop the trailing patch i.e., just return grch37
+        if is_patch_assembly(assembly):
+            assembly_name = re.sub('\\.p[0-9]+$', '', assembly_name.lower())
         assembly_code = re.sub('[^0-9a-zA-Z]+', '', assembly_name.lower())
     return assembly_code
 

ebi_eva_internal_pyutils/mongodb/mongo_database.py

@@ -111,9 +111,7 @@ class MongoDatabase(AppLogger):
             shard_collection_command = f'sh.shardCollection(' \
                                        f'"{self.db_name}.{collection_name}", ' \
                                        f'{shard_key_repr}, {str(shard_key_uniqueness_flag).lower()})'
-            sharding_command = f"mongo --host {self.uri}  " \
-                               f"--eval " \
-                               f"'{shard_collection_command}' "
+            sharding_command = f"mongosh --eval '{shard_collection_command}' {self.uri} "
             sharding_command += self._get_optional_secrets_file_stdin()
             run_command_with_output(f"Sharding collection {collection_name} in the database {self.uri_with_db_name} "
                                     f"with key {shard_key_repr}...", sharding_command,