ebi-eva-common-pyutils 0.6.4__py3-none-any.whl → 0.6.6__py3-none-any.whl

ebi_eva_common_pyutils/assembly_utils.py

@@ -40,16 +40,20 @@ def is_patch_assembly(assembly_accession: str) -> bool:
     return int(xml_assembly[0].text) > 0
 
 
-def retrieve_genbank_assembly_accessions_from_ncbi(assembly_txt):
+def retrieve_genbank_assembly_accessions_from_ncbi(assembly_txt, api_key=None):
     """
     Attempt to find any assembly genebank accession base on a free text search.
     """
     assembly_accessions = set()
     payload = {'db': 'Assembly', 'term': '"{}"'.format(assembly_txt), 'retmode': 'JSON'}
+    if api_key:
+        payload['api_key'] = api_key
     data = requests.get(ESEARCH_URL, params=payload).json()
     if data and data.get('esearchresult', {}).get('idlist'):
         assembly_id_list = data.get('esearchresult').get('idlist')
         payload = {'db': 'Assembly', 'id': ','.join(assembly_id_list), 'retmode': 'JSON'}
+        if api_key:
+            payload['api_key'] = api_key
         summary_list = requests.get(ESUMMARY_URL, params=payload).json()
         for assembly_id in summary_list.get('result', {}).get('uids', []):
             assembly_info = summary_list.get('result').get(assembly_id)
@@ -60,8 +64,8 @@ def retrieve_genbank_assembly_accessions_from_ncbi(assembly_txt):
     return list(assembly_accessions)
 
 
-def retrieve_genbank_equivalent_for_GCF_accession(assembly_accession):
-    genbank_synonyms = retrieve_genbank_assembly_accessions_from_ncbi(assembly_accession)
+def retrieve_genbank_equivalent_for_GCF_accession(assembly_accession, ncbi_api_key=None):
+    genbank_synonyms = retrieve_genbank_assembly_accessions_from_ncbi(assembly_accession, api_key=ncbi_api_key)
     if len(genbank_synonyms) != 1:
         raise ValueError('%s Genbank synonyms found for assembly %s ' % (len(genbank_synonyms), assembly_accession))
     return genbank_synonyms.pop()

ebi_eva_common_pyutils/ncbi_utils.py

@@ -69,8 +69,8 @@ def get_ncbi_taxonomy_dicts_from_ids(taxonomy_ids, api_key=None):
     return taxonomy_dicts
 
 
-def get_ncbi_assembly_name_from_term(term):
-    assembl_dicts = get_ncbi_assembly_dicts_from_term(term)
+def get_ncbi_assembly_name_from_term(term, api_key=None):
+    assembl_dicts = get_ncbi_assembly_dicts_from_term(term, api_key=api_key)
     assembly_names = set([d.get('assemblyname') for d in assembl_dicts])
     if len(assembly_names) > 1:
         # Only keep the one that have the assembly accession as a synonymous and check again
@@ -82,8 +82,10 @@ def get_ncbi_assembly_name_from_term(term):
     return assembly_names.pop() if assembly_names else None
 
 
-def retrieve_species_scientific_name_from_tax_id_ncbi(taxid):
+def retrieve_species_scientific_name_from_tax_id_ncbi(taxid, api_key=None):
     payload = {'db': 'Taxonomy', 'id': taxid}
+    if api_key:
+        payload['api_key'] = api_key
     r = requests.get(efetch_url, params=payload)
     match = re.search('<Rank>(.+?)</Rank>', r.text, re.MULTILINE)
     rank = None
@@ -96,9 +98,9 @@ def retrieve_species_scientific_name_from_tax_id_ncbi(taxid):
         return match.group(1)
 
 
-def get_species_name_from_ncbi(assembly_acc):
+def get_species_name_from_ncbi(assembly_acc, api_key=None):
     # We first need to search for the species associated with the assembly
-    assembly_dicts = get_ncbi_assembly_dicts_from_term(assembly_acc)
+    assembly_dicts = get_ncbi_assembly_dicts_from_term(assembly_acc, api_key=api_key)
     taxids = set([assembly_dict.get('taxid')
         for assembly_dict in assembly_dicts
         if assembly_dict.get('assemblyaccession') == assembly_acc or
@@ -111,5 +113,5 @@ def get_species_name_from_ncbi(assembly_acc):
 
     taxonomy_id = taxids.pop()
 
-    scientific_name = retrieve_species_scientific_name_from_tax_id_ncbi(taxonomy_id)
+    scientific_name = retrieve_species_scientific_name_from_tax_id_ncbi(taxonomy_id, api_key=api_key)
     return scientific_name.replace(' ', '_').lower()

ebi_eva_common_pyutils/network_utils.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import time
 
 import requests
 import subprocess
@@ -43,6 +43,13 @@ def forward_remote_port_to_local_port(remote_host: str, remote_port: int, local_
     port_forward_command = 'ssh -N -L{0}:localhost:{1} {2}'.format(local_port, remote_port, remote_host)
     logger.info("Forwarding port to local port using command: " + port_forward_command)
     proc = subprocess.Popen(port_forward_command.split(" "))
+    time.sleep(5)
+    # Ensure that the process is still running
+    poll = proc.poll()
+    if poll is not None:
+        # The process already completed which mean it most likely crashed
+        logger.error(f'Port Forwarding {remote_host}:{remote_port} -> {local_port} failed!')
+        raise subprocess.CalledProcessError(proc.returncode, proc.args)
     return proc.pid
 
 

{ebi_eva_common_pyutils-0.6.4.dist-info → ebi_eva_common_pyutils-0.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ebi-eva-common-pyutils
-Version: 0.6.4
+Version: 0.6.6
 Summary: EBI EVA - Common Python Utilities
 Home-page: https://github.com/EBIVariation/eva-common-pyutils
 License: Apache

{ebi_eva_common_pyutils-0.6.4.dist-info → ebi_eva_common_pyutils-0.6.6.dist-info}/RECORD

@@ -1,13 +1,13 @@
 ebi_eva_common_pyutils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ebi_eva_common_pyutils/assembly_utils.py,sha256=Lc1bDf0xmi-AF6eyM7pWMgsjnxZmpVHvF3E5aEiKBEI,4018
+ebi_eva_common_pyutils/assembly_utils.py,sha256=CklyCGlCjlFp0e9pugg6kSsh5L0xfCe2qPvA2eLVtn0,4187
 ebi_eva_common_pyutils/command_utils.py,sha256=PtelWWqcC0eOwIVesjwBw3F9KaXRzEE_uAUJhQFZ4l8,2340
 ebi_eva_common_pyutils/common_utils.py,sha256=ty_glvfRa3VGhnpAht4qtVkNNmv-IYfVtO958mY-BaA,1192
 ebi_eva_common_pyutils/config.py,sha256=PtD2SgHf96kk21OA9tVIjEgsDXEFuAU-INy_kfQdoPw,4828
 ebi_eva_common_pyutils/ena_utils.py,sha256=S2MmnWQ_9MJjlkaQY_by1-GGbTyi8SKp8XRcpjWnpZs,1465
 ebi_eva_common_pyutils/file_utils.py,sha256=eIlQKSVKkEjMNX7emrDzaQyQdGvQdb64gnfEhb6uYsE,1375
 ebi_eva_common_pyutils/logger.py,sha256=hT20ktN_oGeB_5ofVfd_aGXB6xYOe0Y5b3EVRYCuFb8,5093
-ebi_eva_common_pyutils/ncbi_utils.py,sha256=sh9E_oDhDtliihJ0-FZuDd7paeidW5GfqRr5vBTo2k4,4859
-ebi_eva_common_pyutils/network_utils.py,sha256=_Qf5oNONA4pUzvxfdk25G-dKuWe44aC_Hewvdp-VZyU,2285
+ebi_eva_common_pyutils/ncbi_utils.py,sha256=AKuNKv4ue7l-6rwuDDGubWEHGuscXWfv-Gg6o_x1hT0,5005
+ebi_eva_common_pyutils/network_utils.py,sha256=iJjs5PPzT1V4CceZnCHOTs711AmpwlDo59wb1XHdTak,2648
 ebi_eva_common_pyutils/assembly/__init__.py,sha256=KSWPwBY5nZj00odxWFntk8Sqg_rw273xH8S5D6Jo-T4,67
 ebi_eva_common_pyutils/assembly/assembly.py,sha256=IEmleROX4ZchPyhINKCuMmET_Ih1Jg4ok-opAKY6Z9A,3142
 ebi_eva_common_pyutils/contig_alias/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -19,11 +19,11 @@ ebi_eva_common_pyutils/taxonomy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 ebi_eva_common_pyutils/taxonomy/taxonomy.py,sha256=p3XV4g3y0hEjyeZ4PwgN7Q3Et9G515ctQkSIo1kdDbU,2259
 ebi_eva_common_pyutils/variation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ebi_eva_common_pyutils/variation/contig_utils.py,sha256=kMNEW_P2yPnd8Xx1tep19hy5ee7ojxz6ZOO1grTQsRQ,5230
-ebi_eva_common_pyutils-0.6.4.data/scripts/archive_directory.py,sha256=0lWJ0ju_AB2ni7lMnJXPFx6U2OdTGbe-WoQs-4BfKOM,4976
+ebi_eva_common_pyutils-0.6.6.data/scripts/archive_directory.py,sha256=0lWJ0ju_AB2ni7lMnJXPFx6U2OdTGbe-WoQs-4BfKOM,4976
 ebi_eva_internal_pyutils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ebi_eva_internal_pyutils/archive_directory.py,sha256=IxVEfh_gaCiT652k0Q_-58fonRusy1yzXu7BCO8yVLo,4989
 ebi_eva_internal_pyutils/config_utils.py,sha256=EGRC5rsmU_ug7OY9-t1UW1XZXRsauSyZB9xPcBux8ts,7909
-ebi_eva_internal_pyutils/metadata_utils.py,sha256=NQVD0_-S1EBb0SnlKX0w31g4Me_8VGXYJTDlMhzje9w,15090
+ebi_eva_internal_pyutils/metadata_utils.py,sha256=t9PcXZdbfjDBP04GJenC4bxm2nOLd8oI_MP9eNe9IBQ,15221
 ebi_eva_internal_pyutils/mongo_utils.py,sha256=YxKHtb5ygDiGLOtEiiAMFCP2ow6FL9Kq0K5R0mWNdXY,3575
 ebi_eva_internal_pyutils/pg_utils.py,sha256=FUQVwiX_7F2-4sSzoaCVX2me0zAqR8nGIj6NW5d304A,4398
 ebi_eva_internal_pyutils/spring_properties.py,sha256=WjPozWtXbAZGNqlgvY6GHps2KFB1rY9OaTs46obW3pM,15265
@@ -31,8 +31,8 @@ ebi_eva_internal_pyutils/mongodb/__init__.py,sha256=0oyTlkYZCV7udlPl09Zl-sDyE3c9
 ebi_eva_internal_pyutils/mongodb/mongo_database.py,sha256=kesaJaaxYFeF_uYZBgL8tbufGKUXll7bXb4WlOS9vKM,9596
 ebi_eva_internal_pyutils/nextflow/__init__.py,sha256=OOiJS8jZOz98q0t77NNog7aI_fFrVxi4kGmiSskuAqM,122
 ebi_eva_internal_pyutils/nextflow/nextflow_pipeline.py,sha256=ew623hhK8jmFLQjJwLZbgBmW9RTiJBEULVqHfIUv_dc,10114
-ebi_eva_common_pyutils-0.6.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ebi_eva_common_pyutils-0.6.4.dist-info/METADATA,sha256=z3BrpkHtxmrrusChh4Oawx8m6tAeRLZpXLlN8pnu3ic,824
-ebi_eva_common_pyutils-0.6.4.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-ebi_eva_common_pyutils-0.6.4.dist-info/top_level.txt,sha256=sXoiqiGU8vlMQpFWDlKrekxhlusk06AhkOH3kSvDT6c,48
-ebi_eva_common_pyutils-0.6.4.dist-info/RECORD,,
+ebi_eva_common_pyutils-0.6.6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ebi_eva_common_pyutils-0.6.6.dist-info/METADATA,sha256=w_We0B6eam8FRWLiTjedqj7rCvQCSf8w16KsC45XMlI,824
+ebi_eva_common_pyutils-0.6.6.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+ebi_eva_common_pyutils-0.6.6.dist-info/top_level.txt,sha256=sXoiqiGU8vlMQpFWDlKrekxhlusk06AhkOH3kSvDT6c,48
+ebi_eva_common_pyutils-0.6.6.dist-info/RECORD,,

ebi_eva_internal_pyutils/metadata_utils.py

@@ -117,10 +117,10 @@ def resolve_existing_variant_warehouse_db_name(metadata_connection_handle, assem
 get_variant_warehouse_db_name_from_assembly_and_taxonomy = resolve_existing_variant_warehouse_db_name
 
 
-def get_assembly_code(metadata_connection_handle, assembly):
+def get_assembly_code(metadata_connection_handle, assembly, ncbi_api_key=None):
     assembly_code = get_assembly_code_from_metadata(metadata_connection_handle, assembly)
     if not assembly_code:
-        assembly_name = get_ncbi_assembly_name_from_term(assembly)
+        assembly_name = get_ncbi_assembly_name_from_term(assembly, api_key=ncbi_api_key)
         # If the assembly is a patch assembly ex: GRCh37.p8, drop the trailing patch i.e., just return grch37
         if is_patch_assembly(assembly):
             assembly_name = re.sub('\\.p[0-9]+$', '', assembly_name.lower())
@@ -141,18 +141,18 @@ def get_taxonomy_code(metadata_connection_handle, taxonomy):
     return taxonomy_code
 
 
-def resolve_variant_warehouse_db_name(metadata_connection_handle, assembly, taxonomy):
+def resolve_variant_warehouse_db_name(metadata_connection_handle, assembly, taxonomy, ncbi_api_key=None):
     """
     Retrieve the database name for this taxonomy/assembly pair whether it exists or not.
     It will use existing taxonomy code or assembly code if available in the metadata database.
     """
     taxonomy_code = get_taxonomy_code(metadata_connection_handle, taxonomy)
-    assembly_code = get_assembly_code(metadata_connection_handle, assembly)
+    assembly_code = get_assembly_code(metadata_connection_handle, assembly, ncbi_api_key=ncbi_api_key)
     return build_variant_warehouse_database_name(taxonomy_code, assembly_code)
 
 
 def insert_new_assembly_and_taxonomy(metadata_connection_handle, assembly_accession, taxonomy_id, eva_species_name=None,
-                                     in_accessioning=True):
+                                     in_accessioning=True, ncbi_api_key=None):
     """
     This script adds new assemblies and taxonomies to EVAPRO.
     You can also add the assembly with a different taxonomy if you provide the
@@ -166,11 +166,10 @@ def insert_new_assembly_and_taxonomy(metadata_connection_handle, assembly_access
         Not required if the taxonomy exists or ENA has a common name available.
     :param in_accessioning: Flag that this assembly is in the accessioning data store.
     """
-    assembly_name = get_ncbi_assembly_name_from_term(assembly_accession)
-
     # check if assembly is already in EVAPRO, adding it if not
     assembly_set_id = get_assembly_set_from_metadata(metadata_connection_handle, taxonomy_id, assembly_accession)
     if assembly_set_id is None:
+        assembly_name = get_ncbi_assembly_name_from_term(assembly_accession, api_key=ncbi_api_key)
         ensure_taxonomy_is_in_evapro(metadata_connection_handle, taxonomy_id, eva_species_name)
         assembly_code = get_assembly_code(metadata_connection_handle, assembly_accession)
         insert_assembly_in_evapro(metadata_connection_handle, taxonomy_id, assembly_accession, assembly_name, assembly_code)