ebi-eva-common-pyutils 0.6.5__tar.gz → 0.6.7__tar.gz

{ebi_eva_common_pyutils-0.6.5 → ebi_eva_common_pyutils-0.6.7}/CHANGELOG.md

@@ -2,6 +2,18 @@
 Changelog for ebi_eva_common_pyutils
 ===========================
 
+## 0.6.7 (2024-05-15)
+---------------------
+
+- Remove the instance id and add required dummy cutoff day for accessioning and clustering 
+
+
+## 0.6.6 (2024-04-15)
+---------------------
+
+- Allow NCBI function ot use API key
+
+
 ## 0.6.5 (2024-04-02)
 ---------------------
 

{ebi_eva_common_pyutils-0.6.5/ebi_eva_common_pyutils.egg-info → ebi_eva_common_pyutils-0.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ebi_eva_common_pyutils
-Version: 0.6.5
+Version: 0.6.7
 Summary: EBI EVA - Common Python Utilities
 Home-page: https://github.com/EBIVariation/eva-common-pyutils
 License: Apache
@@ -10,13 +10,5 @@ Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Build Tools
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
-License-File: LICENSE
-Requires-Dist: requests
-Requires-Dist: lxml
-Requires-Dist: pyyaml
-Requires-Dist: cached-property
-Requires-Dist: retry
 Provides-Extra: eva-internal
-Requires-Dist: psycopg2-binary; extra == "eva-internal"
-Requires-Dist: pymongo; extra == "eva-internal"
-Requires-Dist: networkx<=2.5; extra == "eva-internal"
+License-File: LICENSE

{ebi_eva_common_pyutils-0.6.5 → ebi_eva_common_pyutils-0.6.7}/ebi_eva_common_pyutils/assembly_utils.py

@@ -40,16 +40,20 @@ def is_patch_assembly(assembly_accession: str) -> bool:
     return int(xml_assembly[0].text) > 0
 
 
-def retrieve_genbank_assembly_accessions_from_ncbi(assembly_txt):
+def retrieve_genbank_assembly_accessions_from_ncbi(assembly_txt, api_key=None):
     """
     Attempt to find any assembly genebank accession base on a free text search.
     """
     assembly_accessions = set()
     payload = {'db': 'Assembly', 'term': '"{}"'.format(assembly_txt), 'retmode': 'JSON'}
+    if api_key:
+        payload['api_key'] = api_key
     data = requests.get(ESEARCH_URL, params=payload).json()
     if data and data.get('esearchresult', {}).get('idlist'):
         assembly_id_list = data.get('esearchresult').get('idlist')
         payload = {'db': 'Assembly', 'id': ','.join(assembly_id_list), 'retmode': 'JSON'}
+        if api_key:
+            payload['api_key'] = api_key
         summary_list = requests.get(ESUMMARY_URL, params=payload).json()
         for assembly_id in summary_list.get('result', {}).get('uids', []):
             assembly_info = summary_list.get('result').get(assembly_id)
@@ -60,8 +64,8 @@ def retrieve_genbank_assembly_accessions_from_ncbi(assembly_txt):
     return list(assembly_accessions)
 
 
-def retrieve_genbank_equivalent_for_GCF_accession(assembly_accession):
-    genbank_synonyms = retrieve_genbank_assembly_accessions_from_ncbi(assembly_accession)
+def retrieve_genbank_equivalent_for_GCF_accession(assembly_accession, ncbi_api_key=None):
+    genbank_synonyms = retrieve_genbank_assembly_accessions_from_ncbi(assembly_accession, api_key=ncbi_api_key)
     if len(genbank_synonyms) != 1:
         raise ValueError('%s Genbank synonyms found for assembly %s ' % (len(genbank_synonyms), assembly_accession))
     return genbank_synonyms.pop()

{ebi_eva_common_pyutils-0.6.5 → ebi_eva_common_pyutils-0.6.7}/ebi_eva_common_pyutils/ncbi_utils.py

@@ -69,8 +69,8 @@ def get_ncbi_taxonomy_dicts_from_ids(taxonomy_ids, api_key=None):
     return taxonomy_dicts
 
 
-def get_ncbi_assembly_name_from_term(term):
-    assembl_dicts = get_ncbi_assembly_dicts_from_term(term)
+def get_ncbi_assembly_name_from_term(term, api_key=None):
+    assembl_dicts = get_ncbi_assembly_dicts_from_term(term, api_key=api_key)
     assembly_names = set([d.get('assemblyname') for d in assembl_dicts])
     if len(assembly_names) > 1:
         # Only keep the one that have the assembly accession as a synonymous and check again
@@ -82,8 +82,10 @@ def get_ncbi_assembly_name_from_term(term):
     return assembly_names.pop() if assembly_names else None
 
 
-def retrieve_species_scientific_name_from_tax_id_ncbi(taxid):
+def retrieve_species_scientific_name_from_tax_id_ncbi(taxid, api_key=None):
     payload = {'db': 'Taxonomy', 'id': taxid}
+    if api_key:
+        payload['api_key'] = api_key
     r = requests.get(efetch_url, params=payload)
     match = re.search('<Rank>(.+?)</Rank>', r.text, re.MULTILINE)
     rank = None
@@ -96,9 +98,9 @@ def retrieve_species_scientific_name_from_tax_id_ncbi(taxid):
         return match.group(1)
 
 
-def get_species_name_from_ncbi(assembly_acc):
+def get_species_name_from_ncbi(assembly_acc, api_key=None):
     # We first need to search for the species associated with the assembly
-    assembly_dicts = get_ncbi_assembly_dicts_from_term(assembly_acc)
+    assembly_dicts = get_ncbi_assembly_dicts_from_term(assembly_acc, api_key=api_key)
     taxids = set([assembly_dict.get('taxid')
         for assembly_dict in assembly_dicts
         if assembly_dict.get('assemblyaccession') == assembly_acc or
@@ -111,5 +113,5 @@ def get_species_name_from_ncbi(assembly_acc):
 
     taxonomy_id = taxids.pop()
 
-    scientific_name = retrieve_species_scientific_name_from_tax_id_ncbi(taxonomy_id)
+    scientific_name = retrieve_species_scientific_name_from_tax_id_ncbi(taxonomy_id, api_key=api_key)
     return scientific_name.replace(' ', '_').lower()

{ebi_eva_common_pyutils-0.6.5 → ebi_eva_common_pyutils-0.6.7/ebi_eva_common_pyutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: ebi_eva_common_pyutils
-Version: 0.6.5
+Name: ebi-eva-common-pyutils
+Version: 0.6.7
 Summary: EBI EVA - Common Python Utilities
 Home-page: https://github.com/EBIVariation/eva-common-pyutils
 License: Apache
@@ -10,13 +10,5 @@ Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Build Tools
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
-License-File: LICENSE
-Requires-Dist: requests
-Requires-Dist: lxml
-Requires-Dist: pyyaml
-Requires-Dist: cached-property
-Requires-Dist: retry
 Provides-Extra: eva-internal
-Requires-Dist: psycopg2-binary; extra == "eva-internal"
-Requires-Dist: pymongo; extra == "eva-internal"
-Requires-Dist: networkx<=2.5; extra == "eva-internal"
+License-File: LICENSE

{ebi_eva_common_pyutils-0.6.5 → ebi_eva_common_pyutils-0.6.7}/ebi_eva_common_pyutils.egg-info/SOURCES.txt

@@ -4,7 +4,6 @@ MANIFEST.in
 README.md
 setup.cfg
 setup.py
-/tmp/ebi_eva_common_pyutils-v0.6.5-999yjogj/gitclone/ebi_eva_internal_pyutils/archive_directory.py
 ebi_eva_common_pyutils/__init__.py
 ebi_eva_common_pyutils/assembly_utils.py
 ebi_eva_common_pyutils/command_utils.py
@@ -41,5 +40,4 @@ ebi_eva_internal_pyutils/spring_properties.py
 ebi_eva_internal_pyutils/mongodb/__init__.py
 ebi_eva_internal_pyutils/mongodb/mongo_database.py
 ebi_eva_internal_pyutils/nextflow/__init__.py
-ebi_eva_internal_pyutils/nextflow/nextflow_pipeline.py
-tests/test_common.py
+ebi_eva_internal_pyutils/nextflow/nextflow_pipeline.py

{ebi_eva_common_pyutils-0.6.5 → ebi_eva_common_pyutils-0.6.7}/ebi_eva_internal_pyutils/metadata_utils.py

@@ -117,10 +117,10 @@ def resolve_existing_variant_warehouse_db_name(metadata_connection_handle, assem
 get_variant_warehouse_db_name_from_assembly_and_taxonomy = resolve_existing_variant_warehouse_db_name
 
 
-def get_assembly_code(metadata_connection_handle, assembly):
+def get_assembly_code(metadata_connection_handle, assembly, ncbi_api_key=None):
     assembly_code = get_assembly_code_from_metadata(metadata_connection_handle, assembly)
     if not assembly_code:
-        assembly_name = get_ncbi_assembly_name_from_term(assembly)
+        assembly_name = get_ncbi_assembly_name_from_term(assembly, api_key=ncbi_api_key)
         # If the assembly is a patch assembly ex: GRCh37.p8, drop the trailing patch i.e., just return grch37
         if is_patch_assembly(assembly):
             assembly_name = re.sub('\\.p[0-9]+$', '', assembly_name.lower())
@@ -141,18 +141,18 @@ def get_taxonomy_code(metadata_connection_handle, taxonomy):
     return taxonomy_code
 
 
-def resolve_variant_warehouse_db_name(metadata_connection_handle, assembly, taxonomy):
+def resolve_variant_warehouse_db_name(metadata_connection_handle, assembly, taxonomy, ncbi_api_key=None):
     """
     Retrieve the database name for this taxonomy/assembly pair whether it exists or not.
     It will use existing taxonomy code or assembly code if available in the metadata database.
     """
     taxonomy_code = get_taxonomy_code(metadata_connection_handle, taxonomy)
-    assembly_code = get_assembly_code(metadata_connection_handle, assembly)
+    assembly_code = get_assembly_code(metadata_connection_handle, assembly, ncbi_api_key=ncbi_api_key)
     return build_variant_warehouse_database_name(taxonomy_code, assembly_code)
 
 
 def insert_new_assembly_and_taxonomy(metadata_connection_handle, assembly_accession, taxonomy_id, eva_species_name=None,
-                                     in_accessioning=True):
+                                     in_accessioning=True, ncbi_api_key=None):
     """
     This script adds new assemblies and taxonomies to EVAPRO.
     You can also add the assembly with a different taxonomy if you provide the
@@ -166,11 +166,10 @@ def insert_new_assembly_and_taxonomy(metadata_connection_handle, assembly_access
         Not required if the taxonomy exists or ENA has a common name available.
     :param in_accessioning: Flag that this assembly is in the accessioning data store.
     """
-    assembly_name = get_ncbi_assembly_name_from_term(assembly_accession)
-
     # check if assembly is already in EVAPRO, adding it if not
     assembly_set_id = get_assembly_set_from_metadata(metadata_connection_handle, taxonomy_id, assembly_accession)
     if assembly_set_id is None:
+        assembly_name = get_ncbi_assembly_name_from_term(assembly_accession, api_key=ncbi_api_key)
         ensure_taxonomy_is_in_evapro(metadata_connection_handle, taxonomy_id, eva_species_name)
         assembly_code = get_assembly_code(metadata_connection_handle, assembly_accession)
         insert_assembly_in_evapro(metadata_connection_handle, taxonomy_id, assembly_accession, assembly_name, assembly_code)

{ebi_eva_common_pyutils-0.6.5 → ebi_eva_common_pyutils-0.6.7}/ebi_eva_internal_pyutils/spring_properties.py

@@ -117,10 +117,9 @@ class SpringPropertiesGenerator:
         merge = {**self._common_properties(read_preference=read_preference, chunk_size=chunk_size), **props}
         return merge
 
-    def _common_accessioning_clustering_properties(self, *, instance='', assembly_accession, read_preference, chunk_size):
+    def _common_accessioning_clustering_properties(self, *, assembly_accession, read_preference, chunk_size):
         """Properties common to accessioning and clustering pipelines."""
         props = {
-            'accessioning.instanceId': self._format_str('instance-{0}', instance),
             'accessioning.submitted.categoryId': 'ss',
             'accessioning.clustered.categoryId': 'rs',
             'accessioning.monotonic.ss.blockSize': 100000,
@@ -129,16 +128,18 @@ class SpringPropertiesGenerator:
             'accessioning.monotonic.rs.blockSize': 100000,
             'accessioning.monotonic.rs.blockStartValue': 3000000000,
             'accessioning.monotonic.rs.nextBlockInterval': 1000000000,
+            # This value is not used but is required to create beans in Java
+            'recovery.cutoff.days': 9999999
         }
         merge = {**self._common_accessioning_properties(assembly_accession, read_preference, chunk_size), **props}
         return merge
 
-    def get_accessioning_properties(self, *, instance=None, target_assembly=None, fasta=None, assembly_report=None,
+    def get_accessioning_properties(self, *, target_assembly=None, fasta=None, assembly_report=None,
                                     project_accession=None, aggregation='BASIC', taxonomy_accession=None,
                                     vcf_file='', output_vcf='', chunk_size=100):
         """Properties for accessioning pipeline."""
         return self._format(
-            self._common_accessioning_clustering_properties(instance=instance, assembly_accession=target_assembly,
+            self._common_accessioning_clustering_properties(assembly_accession=target_assembly,
                                                             read_preference='secondaryPreferred', chunk_size=chunk_size),
             {
                 'spring.batch.job.names': 'CREATE_SUBSNP_ACCESSION_JOB',
@@ -154,21 +155,20 @@ class SpringPropertiesGenerator:
             },
         )
 
-    def get_clustering_properties(self, *, instance=None, read_preference='primary',
-                                  job_name=None, source_assembly='', target_assembly='', rs_report_path='', projects='',
+    def get_clustering_properties(self, *, read_preference='primary', job_name=None, source_assembly='',
+                                  target_assembly='', rs_report_path='', projects='',
                                   project_accession='', vcf=''):
         """Properties common to all clustering pipelines, though not all are always used."""
         return self._format(
-            self._common_accessioning_clustering_properties(instance=instance, assembly_accession=target_assembly,
-                                                            read_preference=read_preference, chunk_size=100,
-                                                            ),
+            self._common_accessioning_clustering_properties(assembly_accession=target_assembly,
+                                                            read_preference=read_preference, chunk_size=100),
             {
                 'spring.batch.job.names': job_name,
                 'parameters.remappedFrom': source_assembly,
                 'parameters.projects': projects,
                 'parameters.projectAccession': project_accession,
                 'parameters.vcf': vcf,
-                'parameters.rsReportPath': rs_report_path
+                'parameters.rsReportPath': rs_report_path,
             }
         )
 

{ebi_eva_common_pyutils-0.6.5 → ebi_eva_common_pyutils-0.6.7}/setup.py

@@ -7,7 +7,7 @@ setup(
     name='ebi_eva_common_pyutils',
     scripts=[os.path.join(os.path.dirname(__file__), 'ebi_eva_internal_pyutils', 'archive_directory.py')],
     packages=find_packages(),
-    version='0.6.5',
+    version='0.6.7',
     license='Apache',
     description='EBI EVA - Common Python Utilities',
     url='https://github.com/EBIVariation/eva-common-pyutils',

ebi_eva_common_pyutils-0.6.5/tests/test_common.py

@@ -1,22 +0,0 @@
-import os
-from unittest import TestCase
-
-from ebi_eva_common_pyutils.common_utils import merge_two_dicts, pretty_print
-
-
-class TestCommon(TestCase):
-
-    resources_folder = os.path.join(os.path.dirname(__file__), 'resources')
-
-
-class TestCommonUtils(TestCase):
-
-    def test_merge_two_dicts(self):
-        d1 = {'a': 1, 'b': 2, 'c': 3}
-        d2 = {'d': 4, 'a': 5, 'e': 6}
-        assert merge_two_dicts(d1, d2) == {'a': 5, 'b': 2, 'c': 3, 'd': 4, 'e': 6}
-        assert merge_two_dicts(d2, d1) == {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 6}
-
-    def test_pretty_print(self):
-        pretty_print(['Header 1', 'Long Header 2'],
-                     [['row1 cell 1', 'row1 cell 2'], ['row2 cell 1', 'Super long row2 cell 2']])