wbtools 3.1.0__tar.gz → 3.3.0__tar.gz

{wbtools-3.1.0 → wbtools-3.3.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: wbtools
-Version: 3.1.0
+Version: 3.3.0
 Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
-Home-page: https://github.com/WormBase/wbtools
-Author: Valerio Arnaboldi
-Author-email: valearna@caltech.edu
+Author-email: Valerio Arnaboldi <valearna@caltech.edu>
+Project-URL: Homepage, https://github.com/WormBase/wbtools
+Project-URL: Repository, https://github.com/WormBase/wbtools
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -16,14 +16,15 @@ Requires-Dist: numpy~=1.19.2
 Requires-Dist: fabric~=2.5.0
 Requires-Dist: gensim~=3.8.3
 Requires-Dist: nltk~=3.5
-Requires-Dist: setuptools~=50.3.2
 Requires-Dist: regex~=2020.10.28
 Requires-Dist: pdfminer.six==20201018
 Requires-Dist: pytz~=2021.1
 Requires-Dist: pandas~=1.3.3
 Requires-Dist: requests~=2.31.0
 Requires-Dist: python-dateutil~=2.8.2
+Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: grobid-client
+Requires-Dist: agr-cognito-py
 
 # WBtools
 > Interface to WormBase curation database and Text Mining functions

wbtools-3.3.0/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "wbtools"
+version = "3.3.0"
+authors = [
+    {name = "Valerio Arnaboldi", email = "valearna@caltech.edu"},
+]
+description = "Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions"
+readme = "README.md"
+requires-python = ">=3.6"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "psycopg2-binary",
+    "numpy~=1.19.2",
+    "fabric~=2.5.0",
+    "gensim~=3.8.3",
+    "nltk~=3.5",
+    "regex~=2020.10.28",
+    "pdfminer.six==20201018",
+    "pytz~=2021.1",
+    "pandas~=1.3.3",
+    "requests~=2.31.0",
+    "python-dateutil~=2.8.2",
+    "python-dotenv~=1.0.0",
+    "grobid-client",
+    "agr-cognito-py",
+]
+
+[project.urls]
+Homepage = "https://github.com/WormBase/wbtools"
+Repository = "https://github.com/WormBase/wbtools"

{wbtools-3.1.0 → wbtools-3.3.0}/wbtools/literature/paper.py

@@ -26,7 +26,7 @@ from wbtools.lib.nlp.entity_extraction.email_addresses import get_email_addresse
 from wbtools.lib.nlp.text_preprocessing import preprocess, get_documents_from_text, PaperSections
 from wbtools.lib.timeout import timeout
 from wbtools.literature.person import WBAuthor
-from wbtools.utils.okta_utils import get_authentication_token, generate_headers
+from wbtools.utils.auth_utils import get_authentication_token, generate_headers
 
 logger = logging.getLogger(__name__)
 
@@ -193,6 +193,7 @@ class WBPaper(object):
             blue_api_base_url = os.environ.get('API_SERVER', "literature-rest.alliancegenome.org")
             all_reffiles_for_pap_api = f'https://{blue_api_base_url}/reference/referencefile/show_all/{self.agr_curie}'
             request = urllib.request.Request(url=all_reffiles_for_pap_api)
+            request.add_header("Authorization", f"Bearer {get_authentication_token()}")
             request.add_header("Content-type", "application/json")
             request.add_header("Accept", "application/json")
             added_ref_files = 0
@@ -227,7 +228,9 @@ class WBPaper(object):
             raise Exception("PaperDBManager not set")
         # Get Alliance reference info from WBPaperID
         ref_info_from_xref_api = f"https://{ABC_API}/reference/by_cross_reference/WB:WBPaper{self.paper_id}"
-        ref_info: dict = get_data_from_url(ref_info_from_xref_api)
+        token = get_authentication_token()
+        headers = generate_headers(token)
+        ref_info: dict = get_data_from_url(ref_info_from_xref_api, headers)
         if ref_info:
             self.abstract = ref_info["abstract"]
             self.title = ref_info["title"]

{wbtools-3.1.0 → wbtools-3.3.0}/wbtools/utils/alliance.py

@@ -3,7 +3,7 @@ import logging
 import urllib.request
 
 from wbtools.lib.nlp.common import EntityType
-from wbtools.utils.okta_utils import get_authentication_token
+from wbtools.utils.auth_utils import get_authentication_token
 
 
 logger = logging.getLogger(__name__)

wbtools-3.3.0/wbtools/utils/auth_utils.py

@@ -0,0 +1,21 @@
+"""
+Authentication utilities for AGR API access.
+
+This module provides authentication token generation using AWS Cognito
+via the agr_cognito_py library. The functions are re-exported for
+backward compatibility with code that was using the old Okta-based
+authentication.
+
+Required environment variables for Cognito authentication:
+- COGNITO_ADMIN_CLIENT_ID
+- COGNITO_ADMIN_CLIENT_SECRET
+- COGNITO_TOKEN_URL
+- COGNITO_ADMIN_SCOPE
+"""
+
+from agr_cognito_py import (
+    get_admin_token as get_authentication_token,
+    generate_headers
+)
+
+__all__ = ['get_authentication_token', 'generate_headers']

{wbtools-3.1.0 → wbtools-3.3.0}/wbtools.egg-info/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: wbtools
-Version: 3.1.0
+Version: 3.3.0
 Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
-Home-page: https://github.com/WormBase/wbtools
-Author: Valerio Arnaboldi
-Author-email: valearna@caltech.edu
+Author-email: Valerio Arnaboldi <valearna@caltech.edu>
+Project-URL: Homepage, https://github.com/WormBase/wbtools
+Project-URL: Repository, https://github.com/WormBase/wbtools
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -16,14 +16,15 @@ Requires-Dist: numpy~=1.19.2
 Requires-Dist: fabric~=2.5.0
 Requires-Dist: gensim~=3.8.3
 Requires-Dist: nltk~=3.5
-Requires-Dist: setuptools~=50.3.2
 Requires-Dist: regex~=2020.10.28
 Requires-Dist: pdfminer.six==20201018
 Requires-Dist: pytz~=2021.1
 Requires-Dist: pandas~=1.3.3
 Requires-Dist: requests~=2.31.0
 Requires-Dist: python-dateutil~=2.8.2
+Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: grobid-client
+Requires-Dist: agr-cognito-py
 
 # WBtools
 > Interface to WormBase curation database and Text Mining functions

{wbtools-3.1.0 → wbtools-3.3.0}/wbtools.egg-info/SOURCES.txt

@@ -1,25 +1,6 @@
 LICENSE
 README.md
-setup.py
-tests/__init__.py
-tests/config_reader.py
-tests/db/__init__.py
-tests/db/test_afp.py
-tests/db/test_antibody.py
-tests/db/test_expression.py
-tests/db/test_generic.py
-tests/lib/__init__.py
-tests/lib/test_literature_index.py
-tests/lib/test_scraping.py
-tests/lib/test_text_preprocessing.py
-tests/lib/test_text_similarity.py
-tests/lib/entity_extraction/__init__.py
-tests/lib/entity_extraction/test_email_addresses.py
-tests/lib/entity_extraction/test_generic.py
-tests/lib/entity_extraction/test_new_variations.py
-tests/literature/__init__.py
-tests/literature/test_corpus.py
-tests/literature/test_paper.py
+pyproject.toml
 wbtools/__init__.py
 wbtools.egg-info/PKG-INFO
 wbtools.egg-info/SOURCES.txt
@@ -58,4 +39,4 @@ wbtools/literature/paper.py
 wbtools/literature/person.py
 wbtools/utils/__init__.py
 wbtools/utils/alliance.py
-wbtools/utils/okta_utils.py
+wbtools/utils/auth_utils.py

{wbtools-3.1.0 → wbtools-3.3.0}/wbtools.egg-info/requires.txt

@@ -3,11 +3,12 @@ numpy~=1.19.2
 fabric~=2.5.0
 gensim~=3.8.3
 nltk~=3.5
-setuptools~=50.3.2
 regex~=2020.10.28
 pdfminer.six==20201018
 pytz~=2021.1
 pandas~=1.3.3
 requests~=2.31.0
 python-dateutil~=2.8.2
+python-dotenv~=1.0.0
 grobid-client
+agr-cognito-py

{wbtools-3.1.0 → wbtools-3.3.0}/wbtools.egg-info/top_level.txt

@@ -1,2 +1 @@
-tests
 wbtools

wbtools-3.1.0/setup.py

@@ -1,38 +0,0 @@
-import setuptools
-
-with open("README.md", "r") as fh:
-    long_description = fh.read()
-
-setuptools.setup(
-    name="wbtools",
-    version="3.1.0",
-    author="Valerio Arnaboldi",
-    author_email="valearna@caltech.edu",
-    description="Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP "
-                "functions",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    url="https://github.com/WormBase/wbtools",
-    packages=setuptools.find_packages(),
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: OS Independent",
-    ],
-    python_requires='>=3.6',
-    install_requires=[
-        'psycopg2-binary',
-        'numpy~=1.19.2',
-        'fabric~=2.5.0',
-        'gensim~=3.8.3',
-        'nltk~=3.5',
-        'setuptools~=50.3.2',
-        'regex~=2020.10.28',
-        'pdfminer.six==20201018',
-        'pytz~=2021.1',
-        'pandas~=1.3.3',
-        'requests~=2.31.0',
-        'python-dateutil~=2.8.2',
-        'grobid-client'
-    ]
-)

wbtools-3.1.0/tests/config_reader.py

@@ -1,23 +0,0 @@
-import configparser
-import os
-
-import dotenv
-
-
-def read_config_file(file_path):
-    config = configparser.ConfigParser()
-    config.read(file_path)
-    return config
-
-
-def read_db_config():
-    return read_config_file(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "local_config", "db.cfg"))
-
-
-def read_tpc_config():
-    return read_config_file(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "local_config",
-                                         "textpresso.cfg"))
-
-
-def load_env_file():
-    dotenv.load_dotenv(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "local_config", ".env"))

wbtools-3.1.0/tests/db/test_afp.py

@@ -1,23 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config
-from wbtools.db.afp import WBAFPDBManager
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestWBAFPDBManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBAFPDBManager(dbname=config["wb_database"]["db_name"],
-                                         user=config["wb_database"]["db_user"],
-                                         password=config["wb_database"]["db_password"],
-                                         host=config["wb_database"]["db_host"])
-
-    def test_get_stats_timeseries(self):
-        monthly_stats = self.db_manager.get_stats_timeseries(bin_period='m')
-        self.assertTrue(len(monthly_stats) > 0)
-        self.assertTrue(len(monthly_stats[0]) == 2)
-        self.assertTrue(len(monthly_stats[0][1]) == 5)

wbtools-3.1.0/tests/db/test_antibody.py

@@ -1,23 +0,0 @@
-import datetime
-import os
-import unittest
-
-from tests.config_reader import read_db_config
-from wbtools.db.antibody import WBAntibodyDBManager
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestWBDBManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBAntibodyDBManager(dbname=config["wb_database"]["db_name"],
-                                              user=config["wb_database"]["db_user"],
-                                              password=config["wb_database"]["db_password"],
-                                              host=config["wb_database"]["db_host"])
-
-    def test_get_antibody_str_values(self):
-        antib_str_values = self.db_manager.get_antibody_str_values(
-            from_date=datetime.datetime.now() - datetime.timedelta(days=365))
-        self.assertTrue(len(antib_str_values) > 0)

wbtools-3.1.0/tests/db/test_expression.py

@@ -1,25 +0,0 @@
-import unittest
-import os
-
-from tests.config_reader import read_db_config
-from wbtools.db.expression import WBExpressionDBManager
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestWBExpressionDBManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBExpressionDBManager(dbname=config["wb_database"]["db_name"],
-                                                user=config["wb_database"]["db_user"],
-                                                password=config["wb_database"]["db_password"],
-                                                host=config["wb_database"]["db_host"])
-
-    def test_get_all_expression_sentences(self):
-        sentences = self.db_manager.get_all_expression_sentences()
-        self.assertTrue(sentences)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/db/test_generic.py

@@ -1,44 +0,0 @@
-import re
-import unittest
-import os
-
-from tests.config_reader import read_db_config
-from wbtools.db.generic import WBGenericDBManager
-from wbtools.lib.nlp.entity_extraction.ntt_extractor import ALL_VAR_REGEX
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestWBDBManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBGenericDBManager(dbname=config["wb_database"]["db_name"],
-                                             user=config["wb_database"]["db_user"],
-                                             password=config["wb_database"]["db_password"],
-                                             host=config["wb_database"]["db_host"])
-
-    def test_get_curated_variations(self):
-        curated_variations = self.db_manager.get_curated_variations()
-        self.assertGreater(len(curated_variations), 0)
-        curated_variations = self.db_manager.get_curated_variations(exclude_id_used_as_name=True)
-        allele_regex = ALL_VAR_REGEX.format(designations=self.db_manager.get_allele_designations())
-        for variation in curated_variations:
-            self.assertTrue(re.match(allele_regex, variation.lower()))
-
-    def test_entity_name_id_maps(self):
-        gene_name_id_map = self.db_manager.get_gene_name_id_map()
-        self.assertTrue(len(gene_name_id_map) > 0)
-
-    def test_get_curated_transgenes(self):
-        all_curated = self.db_manager.get_curated_transgenes(exclude_id_used_as_name=False, exclude_invalid=False)
-        exclude_invalid = self.db_manager.get_curated_transgenes(exclude_id_used_as_name=False, exclude_invalid=True)
-        self.assertGreater(len(all_curated), len(exclude_invalid))
-
-    def test_get_blacklisted_emails(self):
-        blacklisted_emails = self.db_manager.get_blacklisted_email_addresses()
-        self.assertGreater(len(blacklisted_emails), 0)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/entity_extraction/test_email_addresses.py

@@ -1,35 +0,0 @@
-import unittest
-import os
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.lib.nlp.entity_extraction.email_addresses import get_email_addresses_from_text
-from wbtools.literature.corpus import CorpusManager
-
-
-class TestEmailAddresses(unittest.TestCase):
-
-    def test_get_email_addresses_from_text(self):
-        text = "test@email.com is a valid email address. @email is not. test@ is not. Other valid email addresses are:" \
-               " t.test@stat.email.org"
-        addr = get_email_addresses_from_text(text)
-        self.assertEqual(len(addr), 2)
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_get_email_addresses_from_paper(self):
-        config = read_db_config()
-        load_env_file()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=config["wb_database"]["db_name"], db_user=config["wb_database"]["db_user"],
-                                 db_password=config["wb_database"]["db_password"],
-                                 db_host=config["wb_database"]["db_host"],
-                                 paper_ids=['00062455'])
-        email_addresses = get_email_addresses_from_text(cm.get_paper('00062455').get_text_docs(
-            include_supplemental=False, return_concatenated=True))
-        email_addresses_in_wb = cm.get_paper('00062455').get_authors_with_email_address_in_wb()
-        self.assertEqual(len(email_addresses), 3)
-        self.assertGreaterEqual(len(email_addresses_in_wb), 2)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/entity_extraction/test_generic.py

@@ -1,46 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config
-from wbtools.db.generic import WBGenericDBManager
-from wbtools.lib.nlp.common import EntityType
-from wbtools.lib.nlp.entity_extraction.ntt_extractor import NttExtractor
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-class TestGenericNLP(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBGenericDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        self.ntt_extractor = NttExtractor(db_manager=self.db_manager)
-
-    def test_extract_entities(self):
-        text = "  g2 is a new variant"
-        variations = self.ntt_extractor.extract_all_entities_by_type(text, entity_type=EntityType.VARIATION)
-        self.assertEqual(len(variations), 1)
-        self.assertTrue("g2" in variations)
-
-    def test_get_strains_from_text(self):
-        text = "  AB4 is a strain, (N2) is another one, but AAA23 is not a strain "
-        strains = self.ntt_extractor.extract_all_entities_by_type(text, EntityType.STRAIN, match_curated=True)
-        self.assertEqual(len(strains), 2)
-
-    def test_get_species_from_text(self):
-        text = "C. elegans is a model organism. C. elegans, C. elegans and C. elegans. C. elegans (1), C. elegans [2], " \
-               "C. tropicalis, D. melanogaster, Drosophila melanogaster, D.melanogaster"
-        species = self.ntt_extractor.extract_species_regex(text=text,
-                                                           taxon_id_name_map=self.db_manager.get_taxon_id_names_map(),
-                                                           blacklist=["4853"],
-                                                           whitelist=["Homo sapiens"],
-                                                           min_matches=2)
-        self.assertTrue(len(species) == 2 and
-                        "Caenorhabditis elegans" in species and
-                        "Drosophila melanogaster" in species)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/entity_extraction/test_new_variations.py

@@ -1,44 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config
-from wbtools.db.generic import WBGenericDBManager
-from wbtools.lib.nlp.common import EntityType
-from wbtools.lib.nlp.entity_extraction.ntt_extractor import NttExtractor
-from wbtools.lib.nlp.entity_extraction.variations import is_variation_suspicious
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-class TestNLPNewVariations(unittest.TestCase):
-
-    def setUp(self) -> None:
-        self.text = "  g2 is a new variant"
-        config = read_db_config()
-        db_manager = WBGenericDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        self.ntt_extractor = NttExtractor(db_manager=db_manager)
-
-    def test_get_new_variations_from_text(self):
-        variations = self.ntt_extractor.extract_all_entities_by_type(self.text, EntityType.VARIATION,
-                                                                     match_curated=False,
-                                                                     exclude_curated=True)
-        self.assertGreater(len(variations), 0)
-        text = "The result- ing ez72syb1455 animals show the exact same gfp expression pattern as that of the " \
-               "parental ez72 strain and also display no mutant phenotype (Figures S2A–S2C), thereby supporting past " \
-               "evidence [8] that the tra-1b isoform bears no relevant Tra function."
-        variations = self.ntt_extractor.extract_all_entities_by_type(text, EntityType.VARIATION, match_curated=False,
-                                                                     exclude_curated=True)
-        self.assertTrue("ez72syb1455" in variations)
-
-    def test_is_variation_suspicious(self):
-        variations = self.ntt_extractor.extract_all_entities_by_type(self.text, EntityType.VARIATION,
-                                                                     match_curated=False,
-                                                                     exclude_curated=True)
-        is_susp = is_variation_suspicious(variations[0])
-        self.assertEqual(is_susp, "potentially new")
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/test_literature_index.py

@@ -1,29 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_tpc_config
-from wbtools.lib.nlp.literature_index.textpresso import TextpressoLiteratureIndex
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "textpresso.cfg")),
-                 "Textpresso test config file not present")
-class TestTextpressoLiteratureIndex(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_tpc_config()
-        self.lit_index = TextpressoLiteratureIndex(api_url=config["tpc"]["api_base_url"],
-                                                   api_token=config["tpc"]["api_token"], use_cache=True,
-                                                   corpora=[config["tpc"]["corpora"]])
-
-    def test_num_documents(self):
-        self.assertGreater(self.lit_index.num_documents(), 0)
-        self.assertGreater(self.lit_index.num_documents(), 0)
-
-    def test_count_matching_documents(self):
-        count = self.lit_index.count_matching_documents("DREAM complex")
-        self.assertGreater(count, 0)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/test_scraping.py

@@ -1,18 +0,0 @@
-import unittest
-
-from wbtools.lib.scraping import get_cgc_allele_designations, get_cgc_lab_designations
-
-
-class TestScraping(unittest.TestCase):
-
-    def test_get_cgc_allele_designations(self):
-        cgc_designations = get_cgc_allele_designations()
-        self.assertTrue(len(cgc_designations) > 0)
-
-    def test_get_cgc_lab_designations(self):
-        cgc_lab_designations = get_cgc_lab_designations()
-        self.assertTrue(len(cgc_lab_designations) > 0)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/test_text_preprocessing.py

@@ -1,59 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.lib.nlp.common import PaperSections
-from wbtools.lib.nlp.text_preprocessing import remove_sections_from_text, get_documents_from_text, preprocess
-from wbtools.literature.corpus import CorpusManager
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), '../data')
-
-
-class TestTextPreprocessing(unittest.TestCase):
-
-    def setUp(self):
-        config = read_db_config()
-        load_env_file()
-        self.cm = CorpusManager()
-        self.cm.load_from_wb_database(db_name=config["wb_database"]["db_name"],
-                                      db_user=config["wb_database"]["db_user"],
-                                      db_password=config["wb_database"]["db_password"],
-                                      db_host=config["wb_database"]["db_host"],
-                                      paper_ids=['00062455'])
-
-    def test_sectioning(self):
-        text = list(self.cm.corpus.values())[0].get_text_docs()[0]
-        self.assertTrue("REFERENCES" not in remove_sections_from_text(
-            text=text, sections_to_remove=[PaperSections.INTRODUCTION, PaperSections.REFERENCES],
-            must_be_present=[PaperSections.RESULTS]))
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_sectioning_cell_template(self):
-        config = read_db_config()
-        load_env_file()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=config["wb_database"]["db_name"], db_user=config["wb_database"]["db_user"],
-                                 db_password=config["wb_database"]["db_password"],
-                                 db_host=config["wb_database"]["db_host"],
-                                 paper_ids=['00059375'])
-        fulltext = cm.get_paper('00059375').get_text_docs(remove_sections=[PaperSections.REFERENCES],
-                                                          must_be_present=[PaperSections.METHOD, PaperSections.RESULTS])
-        self.assertTrue(fulltext)
-
-    def test_get_documents_from_text(self):
-        docs = get_documents_from_text(list(self.cm.corpus.values())[0].main_text)
-        self.assertEqual(len(docs), 1)
-        docs = get_documents_from_text(list(self.cm.corpus.values())[0].main_text, split_sentences=True)
-        self.assertGreater(len(docs), 10)
-
-    def test_preprocess(self):
-        doc = get_documents_from_text(list(self.cm.corpus.values())[0].main_text)[0]
-        prep_text = preprocess(doc, lower=True, tokenize=True, remove_alpha=True, remove_stopwords=True)
-        self.assertGreater(len(prep_text), 10)
-        self.assertTrue(all([k == k.lower() for k in prep_text]))
-        self.assertTrue("the" not in prep_text)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/test_text_similarity.py

@@ -1,52 +0,0 @@
-import os
-import unittest
-
-from gensim.models import Word2Vec
-from gensim.test.utils import common_texts
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.lib.nlp.common import PaperSections
-from wbtools.lib.nlp.text_similarity import get_softcosine_index, get_similar_documents
-from wbtools.literature.corpus import CorpusManager
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), '../data')
-
-
-class TestTextSimilarity(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        load_env_file()
-        self.cm = CorpusManager()
-        self.cm.load_from_wb_database(db_name=config["wb_database"]["db_name"],
-                                      db_user=config["wb_database"]["db_user"],
-                                      db_password=config["wb_database"]["db_password"],
-                                      db_host=config["wb_database"]["db_host"],
-                                      paper_ids=['00026804', '00026805', '00026806', '00026807', '00026808',
-                                                 '00026809', '00026810', '00026811', '00026812', '00026813',
-                                                 '00026814', '00026815', '00026816', '00026817', '00026818',
-                                                 '00026819', '00026820', '00026821', '00026822', '00026823'])
-
-    def test_soft_cosine_similarity(self):
-        corpus_list_token, idx_paperid_map = self.cm.get_flat_corpus_list_and_idx_paperid_map(
-            split_sentences=False, remove_sections=[PaperSections.REFERENCES], lowercase=True, tokenize=True,
-            remove_stopwords=True, remove_alpha=True)
-        model = Word2Vec(common_texts, min_count=1)
-        docsim_index, dictionary = get_softcosine_index(model=model, corpus_list_token=corpus_list_token)
-        query_docs = [corpus_list_token[0], corpus_list_token[1]]
-        sim_docs = get_similar_documents(docsim_index, dictionary, query_docs, idx_paperid_map)
-        self.assertTrue(idx_paperid_map[0] in [sim_doc.paper_id for sim_doc in sim_docs[0:2]])
-        self.assertTrue(idx_paperid_map[1] in [sim_doc.paper_id for sim_doc in sim_docs[0:2]])
-
-        corpus_list_token, idx_paperid_map = self.cm.get_flat_corpus_list_and_idx_paperid_map(
-            split_sentences=True, remove_sections=[PaperSections.REFERENCES], lowercase=True, tokenize=True,
-            remove_stopwords=True, remove_alpha=True)
-        docsim_index, dictionary = get_softcosine_index(model=model, corpus_list_token=corpus_list_token)
-        query_docs = [corpus_list_token[0], corpus_list_token[1]]
-        sim_docs = get_similar_documents(docsim_index, dictionary, query_docs, idx_paperid_map)
-        self.assertTrue(idx_paperid_map[0] in [sim_doc.paper_id for sim_doc in sim_docs[0:2]])
-        self.assertTrue(idx_paperid_map[1] in [sim_doc.paper_id for sim_doc in sim_docs[0:2]])
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/literature/test_corpus.py

@@ -1,86 +0,0 @@
-import os
-import unittest
-import tempfile
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.lib.nlp.common import PaperSections
-from wbtools.literature.corpus import CorpusManager
-from wbtools.literature.paper import WBPaper
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), '../data', 'text_files')
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestCorpusManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        load_env_file()
-        self.cm = CorpusManager()
-        self.cm.load_from_wb_database(db_host=config["wb_database"]["db_host"],
-                                      db_user=config["wb_database"]["db_user"],
-                                      db_password=config["wb_database"]["db_password"],
-                                      db_name=config["wb_database"]["db_name"], max_num_papers=2)
-
-    def test_get_flat_corpus_list_and_idx_paperid_map(self):
-        flat_list, idx_paperid_map = self.cm.get_flat_corpus_list_and_idx_paperid_map(
-            tokenize=True, remove_stopwords=True, remove_alpha=True)
-        self.assertEqual(len(flat_list), len(idx_paperid_map))
-        self.assertTrue(len(idx_paperid_map[0]) == 8)
-        self.assertEqual(len(set(idx_paperid_map.values())), self.cm.size())
-
-    def test_split_sentences(self):
-        cl1, map1 = self.cm.get_flat_corpus_list_and_idx_paperid_map(tokenize=False, remove_stopwords=False,
-                                                                     remove_alpha=False, split_sentences=True)
-        cl2, map2 = self.cm.get_flat_corpus_list_and_idx_paperid_map(tokenize=True, remove_stopwords=True,
-                                                                     remove_alpha=True, split_sentences=True)
-        self.assertEqual(len(cl1), len(cl2))
-
-    def test_exclude_sections(self):
-        test_list, _ = self.cm.get_flat_corpus_list_and_idx_paperid_map(
-            split_sentences=False, remove_sections=[PaperSections.INTRODUCTION, PaperSections.REFERENCES],
-            must_be_present=[PaperSections.RESULTS], lowercase=False, tokenize=False, remove_stopwords=False,
-            remove_alpha=False)
-        pass
-
-    def test_load_from_wb_database(self):
-        db_config = read_db_config()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=db_config["wb_database"]["db_name"], db_user=db_config["wb_database"]["db_user"],
-                                 db_password=db_config["wb_database"]["db_password"],
-                                 db_host=db_config["wb_database"]["db_host"], max_num_papers=2)
-        self.assertTrue(cm.size() == 2)
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=db_config["wb_database"]["db_name"],
-                                 db_user=db_config["wb_database"]["db_user"],
-                                 db_password=db_config["wb_database"]["db_password"],
-                                 db_host=db_config["wb_database"]["db_host"], max_num_papers=2,
-                                 exclude_temp_pdf=True)
-        self.assertFalse(any([paper.is_temp() for paper in cm.get_all_papers()]))
-
-    def test_load_supplemental(self):
-        db_config = read_db_config()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=db_config["wb_database"]["db_name"],
-                                 db_user=db_config["wb_database"]["db_user"],
-                                 db_password=db_config["wb_database"]["db_password"],
-                                 db_host=db_config["wb_database"]["db_host"],
-                                 paper_ids=["00062512"])
-        self.assertTrue(len(cm.get_paper("00062512").supplemental_docs) > 0)
-
-    # def test_load_from_wb_database_afp(self):
-    #     db_config = read_db_config()
-    #     cm = CorpusManager()
-    #     cm.load_from_wb_database(db_name=db_config["wb_database"]["db_name"],
-    #                              db_user=db_config["wb_database"]["db_user"],
-    #                              db_password=db_config["wb_database"]["db_password"],
-    #                              db_host=db_config["wb_database"]["db_host"],
-    #                              max_num_papers=2,
-    #                              load_curation_info=True, load_afp_info=True,
-    #                              exclude_temp_pdf=True, exclude_afp_processed=True, must_be_autclass_flagged=True)
-    #     self.assertFalse(any([paper.afp_processed for paper in cm.get_all_papers()]))
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/literature/test_paper.py

@@ -1,110 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.db.paper import WBPaperDBManager
-from wbtools.lib.nlp.common import PaperSections
-from wbtools.literature.corpus import CorpusManager
-from wbtools.literature.paper import WBPaper
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), '../data', 'text_files')
-
-
-class TestWBPaper(unittest.TestCase):
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_load_curation_info_from_db(self):
-        config = read_db_config()
-        load_env_file()
-        paper = WBPaper(paper_id="00004161", db_manager=WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"]))
-        paper.paper_id = "00004161"
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        paper.load_curation_info_from_db()
-        self.assertTrue(paper.aut_class_values["seqchange"] == 'high')
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_load_bib_info(self):
-        config = read_db_config()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00004161", db_manager=db_manager)
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        with db_manager:
-            paper.load_bib_info()
-        self.assertGreater(len(paper.authors), 0)
-        self.assertTrue(paper.abstract.startswith("Niemann-Pick type C (NP-C) disease is a progressive"))
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_extract_all_email_addresses_from_text(self):
-        config = read_db_config()
-        load_env_file()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_host=config["wb_database"]["db_host"],
-                                 db_user=config["wb_database"]["db_user"],
-                                 db_password=config["wb_database"]["db_password"],
-                                 db_name=config["wb_database"]["db_name"], paper_ids=['00026804'])
-        addresses = cm.get_paper('00026804').extract_all_email_addresses_from_text()
-        self.assertGreater(len(addresses), 0)
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_pdf2txt_conversion(self):
-        config = read_db_config()
-        load_env_file()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00003969", db_manager=db_manager)
-        paper.load_bib_info()
-        paper.load_text_from_pdf_files()
-        fulltext = paper.get_text_docs()
-        self.assertGreater(len(fulltext), 0)
-        self.assertTrue("u253, u423 deletion in exon 3" in fulltext[0])
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_pdf_table_conversion(self):
-        config = read_db_config()
-        load_env_file()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00059755", db_manager=db_manager)
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        paper.load_text_from_pdf_files()
-        fulltext = paper.get_text_docs()
-        self.assertTrue(fulltext)
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_tokenize_sentences_with_tables(self):
-        config = read_db_config()
-        load_env_file()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00003969", db_manager=db_manager)
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        paper.load_text_from_pdf_files()
-        sentences = paper.get_text_docs(split_sentences=True)
-        self.assertGreater(len(sentences), 0)
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_two_cols_conversion(self):
-        config = read_db_config()
-        load_env_file()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00055367", db_manager=db_manager)
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        paper.load_text_from_pdf_files()
-        fulltext = paper.get_text_docs()
-        self.assertTrue(fulltext)

wbtools-3.1.0/wbtools/utils/okta_utils.py

@@ -1,67 +0,0 @@
-import os
-from json import loads
-from os import environ, path
-import logging
-import logging.config
-from datetime import datetime
-from dateutil.relativedelta import relativedelta
-import requests
-
-
-def generate_headers(token):
-    """
-
-    :param token:
-    :return:
-    """
-
-    authorization = 'Bearer ' + token
-    headers = {
-        'Authorization': authorization,
-        'Content-Type': 'application/json',
-        'Accept': 'application/json'
-    }
-    return headers
-
-
-def update_okta_token():  # pragma: no cover
-    """
-
-    :return:
-    """
-
-    url = f"https://{os.environ.get('OKTA_DOMAIN')}/v1/token"
-    headers = {
-        'Content-Type': 'application/x-www-form-urlencoded',
-        'Accept': 'application/json'
-    }
-    data_dict = dict()
-    data_dict['grant_type'] = 'client_credentials'
-    data_dict['client_id'] = environ.get('OKTA_CLIENT_ID')
-    data_dict['client_secret'] = environ.get('OKTA_CLIENT_SECRET')
-    data_dict['scope'] = 'admin'
-    post_return = requests.post(url, headers=headers, data=data_dict)
-    logging.warning(post_return.text)
-    response_dict = loads(post_return.text)
-    token = response_dict['access_token']
-    okta_file = environ.get('TMP_PATH') + 'okta_token'
-    os.makedirs(environ.get('TMP_PATH'), exist_ok=True)
-    with open(okta_file, 'w') as okta_fh:
-        okta_fh.write("%s" % token)
-    return token
-
-
-def get_authentication_token():  # pragma: no cover
-    okta_file = environ.get('TMP_PATH') + 'okta_token'
-    token = ''
-    if path.isfile(okta_file):
-        one_day_ago = datetime.now() - relativedelta(days=1)
-        file_time = datetime.fromtimestamp(path.getmtime(okta_file))
-        if file_time > one_day_ago:
-            with open(okta_file, 'r') as okta_fh:
-                token = okta_fh.read().replace("\n", "")
-        else:
-            token = update_okta_token()
-    else:
-        token = update_okta_token()
-    return token