wbtools 3.1.0__tar.gz → 3.2.0__tar.gz

{wbtools-3.1.0 → wbtools-3.2.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: wbtools
-Version: 3.1.0
+Version: 3.2.0
 Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
-Home-page: https://github.com/WormBase/wbtools
-Author: Valerio Arnaboldi
-Author-email: valearna@caltech.edu
+Author-email: Valerio Arnaboldi <valearna@caltech.edu>
+Project-URL: Homepage, https://github.com/WormBase/wbtools
+Project-URL: Repository, https://github.com/WormBase/wbtools
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -16,13 +16,13 @@ Requires-Dist: numpy~=1.19.2
 Requires-Dist: fabric~=2.5.0
 Requires-Dist: gensim~=3.8.3
 Requires-Dist: nltk~=3.5
-Requires-Dist: setuptools~=50.3.2
 Requires-Dist: regex~=2020.10.28
 Requires-Dist: pdfminer.six==20201018
 Requires-Dist: pytz~=2021.1
 Requires-Dist: pandas~=1.3.3
 Requires-Dist: requests~=2.31.0
 Requires-Dist: python-dateutil~=2.8.2
+Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: grobid-client
 
 # WBtools

wbtools-3.2.0/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "wbtools"
+version = "3.2.0"
+authors = [
+    {name = "Valerio Arnaboldi", email = "valearna@caltech.edu"},
+]
+description = "Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions"
+readme = "README.md"
+requires-python = ">=3.6"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "psycopg2-binary",
+    "numpy~=1.19.2",
+    "fabric~=2.5.0",
+    "gensim~=3.8.3",
+    "nltk~=3.5",
+    "regex~=2020.10.28",
+    "pdfminer.six==20201018",
+    "pytz~=2021.1",
+    "pandas~=1.3.3",
+    "requests~=2.31.0",
+    "python-dateutil~=2.8.2",
+    "python-dotenv~=1.0.0",
+    "grobid-client",
+]
+
+[project.urls]
+Homepage = "https://github.com/WormBase/wbtools"
+Repository = "https://github.com/WormBase/wbtools"

{wbtools-3.1.0 → wbtools-3.2.0}/wbtools.egg-info/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: wbtools
-Version: 3.1.0
+Version: 3.2.0
 Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
-Home-page: https://github.com/WormBase/wbtools
-Author: Valerio Arnaboldi
-Author-email: valearna@caltech.edu
+Author-email: Valerio Arnaboldi <valearna@caltech.edu>
+Project-URL: Homepage, https://github.com/WormBase/wbtools
+Project-URL: Repository, https://github.com/WormBase/wbtools
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -16,13 +16,13 @@ Requires-Dist: numpy~=1.19.2
 Requires-Dist: fabric~=2.5.0
 Requires-Dist: gensim~=3.8.3
 Requires-Dist: nltk~=3.5
-Requires-Dist: setuptools~=50.3.2
 Requires-Dist: regex~=2020.10.28
 Requires-Dist: pdfminer.six==20201018
 Requires-Dist: pytz~=2021.1
 Requires-Dist: pandas~=1.3.3
 Requires-Dist: requests~=2.31.0
 Requires-Dist: python-dateutil~=2.8.2
+Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: grobid-client
 
 # WBtools

{wbtools-3.1.0 → wbtools-3.2.0}/wbtools.egg-info/SOURCES.txt

@@ -1,25 +1,6 @@
 LICENSE
 README.md
-setup.py
-tests/__init__.py
-tests/config_reader.py
-tests/db/__init__.py
-tests/db/test_afp.py
-tests/db/test_antibody.py
-tests/db/test_expression.py
-tests/db/test_generic.py
-tests/lib/__init__.py
-tests/lib/test_literature_index.py
-tests/lib/test_scraping.py
-tests/lib/test_text_preprocessing.py
-tests/lib/test_text_similarity.py
-tests/lib/entity_extraction/__init__.py
-tests/lib/entity_extraction/test_email_addresses.py
-tests/lib/entity_extraction/test_generic.py
-tests/lib/entity_extraction/test_new_variations.py
-tests/literature/__init__.py
-tests/literature/test_corpus.py
-tests/literature/test_paper.py
+pyproject.toml
 wbtools/__init__.py
 wbtools.egg-info/PKG-INFO
 wbtools.egg-info/SOURCES.txt

{wbtools-3.1.0 → wbtools-3.2.0}/wbtools.egg-info/requires.txt

@@ -3,11 +3,11 @@ numpy~=1.19.2
 fabric~=2.5.0
 gensim~=3.8.3
 nltk~=3.5
-setuptools~=50.3.2
 regex~=2020.10.28
 pdfminer.six==20201018
 pytz~=2021.1
 pandas~=1.3.3
 requests~=2.31.0
 python-dateutil~=2.8.2
+python-dotenv~=1.0.0
 grobid-client

{wbtools-3.1.0 → wbtools-3.2.0}/wbtools.egg-info/top_level.txt

@@ -1,2 +1 @@
-tests
 wbtools

wbtools-3.1.0/setup.py

@@ -1,38 +0,0 @@
-import setuptools
-
-with open("README.md", "r") as fh:
-    long_description = fh.read()
-
-setuptools.setup(
-    name="wbtools",
-    version="3.1.0",
-    author="Valerio Arnaboldi",
-    author_email="valearna@caltech.edu",
-    description="Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP "
-                "functions",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    url="https://github.com/WormBase/wbtools",
-    packages=setuptools.find_packages(),
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: OS Independent",
-    ],
-    python_requires='>=3.6',
-    install_requires=[
-        'psycopg2-binary',
-        'numpy~=1.19.2',
-        'fabric~=2.5.0',
-        'gensim~=3.8.3',
-        'nltk~=3.5',
-        'setuptools~=50.3.2',
-        'regex~=2020.10.28',
-        'pdfminer.six==20201018',
-        'pytz~=2021.1',
-        'pandas~=1.3.3',
-        'requests~=2.31.0',
-        'python-dateutil~=2.8.2',
-        'grobid-client'
-    ]
-)

wbtools-3.1.0/tests/config_reader.py

@@ -1,23 +0,0 @@
-import configparser
-import os
-
-import dotenv
-
-
-def read_config_file(file_path):
-    config = configparser.ConfigParser()
-    config.read(file_path)
-    return config
-
-
-def read_db_config():
-    return read_config_file(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "local_config", "db.cfg"))
-
-
-def read_tpc_config():
-    return read_config_file(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "local_config",
-                                         "textpresso.cfg"))
-
-
-def load_env_file():
-    dotenv.load_dotenv(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "local_config", ".env"))

wbtools-3.1.0/tests/db/test_afp.py

@@ -1,23 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config
-from wbtools.db.afp import WBAFPDBManager
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestWBAFPDBManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBAFPDBManager(dbname=config["wb_database"]["db_name"],
-                                         user=config["wb_database"]["db_user"],
-                                         password=config["wb_database"]["db_password"],
-                                         host=config["wb_database"]["db_host"])
-
-    def test_get_stats_timeseries(self):
-        monthly_stats = self.db_manager.get_stats_timeseries(bin_period='m')
-        self.assertTrue(len(monthly_stats) > 0)
-        self.assertTrue(len(monthly_stats[0]) == 2)
-        self.assertTrue(len(monthly_stats[0][1]) == 5)

wbtools-3.1.0/tests/db/test_antibody.py

@@ -1,23 +0,0 @@
-import datetime
-import os
-import unittest
-
-from tests.config_reader import read_db_config
-from wbtools.db.antibody import WBAntibodyDBManager
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestWBDBManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBAntibodyDBManager(dbname=config["wb_database"]["db_name"],
-                                              user=config["wb_database"]["db_user"],
-                                              password=config["wb_database"]["db_password"],
-                                              host=config["wb_database"]["db_host"])
-
-    def test_get_antibody_str_values(self):
-        antib_str_values = self.db_manager.get_antibody_str_values(
-            from_date=datetime.datetime.now() - datetime.timedelta(days=365))
-        self.assertTrue(len(antib_str_values) > 0)

wbtools-3.1.0/tests/db/test_expression.py

@@ -1,25 +0,0 @@
-import unittest
-import os
-
-from tests.config_reader import read_db_config
-from wbtools.db.expression import WBExpressionDBManager
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestWBExpressionDBManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBExpressionDBManager(dbname=config["wb_database"]["db_name"],
-                                                user=config["wb_database"]["db_user"],
-                                                password=config["wb_database"]["db_password"],
-                                                host=config["wb_database"]["db_host"])
-
-    def test_get_all_expression_sentences(self):
-        sentences = self.db_manager.get_all_expression_sentences()
-        self.assertTrue(sentences)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/db/test_generic.py

@@ -1,44 +0,0 @@
-import re
-import unittest
-import os
-
-from tests.config_reader import read_db_config
-from wbtools.db.generic import WBGenericDBManager
-from wbtools.lib.nlp.entity_extraction.ntt_extractor import ALL_VAR_REGEX
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestWBDBManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBGenericDBManager(dbname=config["wb_database"]["db_name"],
-                                             user=config["wb_database"]["db_user"],
-                                             password=config["wb_database"]["db_password"],
-                                             host=config["wb_database"]["db_host"])
-
-    def test_get_curated_variations(self):
-        curated_variations = self.db_manager.get_curated_variations()
-        self.assertGreater(len(curated_variations), 0)
-        curated_variations = self.db_manager.get_curated_variations(exclude_id_used_as_name=True)
-        allele_regex = ALL_VAR_REGEX.format(designations=self.db_manager.get_allele_designations())
-        for variation in curated_variations:
-            self.assertTrue(re.match(allele_regex, variation.lower()))
-
-    def test_entity_name_id_maps(self):
-        gene_name_id_map = self.db_manager.get_gene_name_id_map()
-        self.assertTrue(len(gene_name_id_map) > 0)
-
-    def test_get_curated_transgenes(self):
-        all_curated = self.db_manager.get_curated_transgenes(exclude_id_used_as_name=False, exclude_invalid=False)
-        exclude_invalid = self.db_manager.get_curated_transgenes(exclude_id_used_as_name=False, exclude_invalid=True)
-        self.assertGreater(len(all_curated), len(exclude_invalid))
-
-    def test_get_blacklisted_emails(self):
-        blacklisted_emails = self.db_manager.get_blacklisted_email_addresses()
-        self.assertGreater(len(blacklisted_emails), 0)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/entity_extraction/test_email_addresses.py

@@ -1,35 +0,0 @@
-import unittest
-import os
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.lib.nlp.entity_extraction.email_addresses import get_email_addresses_from_text
-from wbtools.literature.corpus import CorpusManager
-
-
-class TestEmailAddresses(unittest.TestCase):
-
-    def test_get_email_addresses_from_text(self):
-        text = "test@email.com is a valid email address. @email is not. test@ is not. Other valid email addresses are:" \
-               " t.test@stat.email.org"
-        addr = get_email_addresses_from_text(text)
-        self.assertEqual(len(addr), 2)
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_get_email_addresses_from_paper(self):
-        config = read_db_config()
-        load_env_file()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=config["wb_database"]["db_name"], db_user=config["wb_database"]["db_user"],
-                                 db_password=config["wb_database"]["db_password"],
-                                 db_host=config["wb_database"]["db_host"],
-                                 paper_ids=['00062455'])
-        email_addresses = get_email_addresses_from_text(cm.get_paper('00062455').get_text_docs(
-            include_supplemental=False, return_concatenated=True))
-        email_addresses_in_wb = cm.get_paper('00062455').get_authors_with_email_address_in_wb()
-        self.assertEqual(len(email_addresses), 3)
-        self.assertGreaterEqual(len(email_addresses_in_wb), 2)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/entity_extraction/test_generic.py

@@ -1,46 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config
-from wbtools.db.generic import WBGenericDBManager
-from wbtools.lib.nlp.common import EntityType
-from wbtools.lib.nlp.entity_extraction.ntt_extractor import NttExtractor
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-class TestGenericNLP(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        self.db_manager = WBGenericDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        self.ntt_extractor = NttExtractor(db_manager=self.db_manager)
-
-    def test_extract_entities(self):
-        text = "  g2 is a new variant"
-        variations = self.ntt_extractor.extract_all_entities_by_type(text, entity_type=EntityType.VARIATION)
-        self.assertEqual(len(variations), 1)
-        self.assertTrue("g2" in variations)
-
-    def test_get_strains_from_text(self):
-        text = "  AB4 is a strain, (N2) is another one, but AAA23 is not a strain "
-        strains = self.ntt_extractor.extract_all_entities_by_type(text, EntityType.STRAIN, match_curated=True)
-        self.assertEqual(len(strains), 2)
-
-    def test_get_species_from_text(self):
-        text = "C. elegans is a model organism. C. elegans, C. elegans and C. elegans. C. elegans (1), C. elegans [2], " \
-               "C. tropicalis, D. melanogaster, Drosophila melanogaster, D.melanogaster"
-        species = self.ntt_extractor.extract_species_regex(text=text,
-                                                           taxon_id_name_map=self.db_manager.get_taxon_id_names_map(),
-                                                           blacklist=["4853"],
-                                                           whitelist=["Homo sapiens"],
-                                                           min_matches=2)
-        self.assertTrue(len(species) == 2 and
-                        "Caenorhabditis elegans" in species and
-                        "Drosophila melanogaster" in species)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/entity_extraction/test_new_variations.py

@@ -1,44 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config
-from wbtools.db.generic import WBGenericDBManager
-from wbtools.lib.nlp.common import EntityType
-from wbtools.lib.nlp.entity_extraction.ntt_extractor import NttExtractor
-from wbtools.lib.nlp.entity_extraction.variations import is_variation_suspicious
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-class TestNLPNewVariations(unittest.TestCase):
-
-    def setUp(self) -> None:
-        self.text = "  g2 is a new variant"
-        config = read_db_config()
-        db_manager = WBGenericDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        self.ntt_extractor = NttExtractor(db_manager=db_manager)
-
-    def test_get_new_variations_from_text(self):
-        variations = self.ntt_extractor.extract_all_entities_by_type(self.text, EntityType.VARIATION,
-                                                                     match_curated=False,
-                                                                     exclude_curated=True)
-        self.assertGreater(len(variations), 0)
-        text = "The result- ing ez72syb1455 animals show the exact same gfp expression pattern as that of the " \
-               "parental ez72 strain and also display no mutant phenotype (Figures S2A–S2C), thereby supporting past " \
-               "evidence [8] that the tra-1b isoform bears no relevant Tra function."
-        variations = self.ntt_extractor.extract_all_entities_by_type(text, EntityType.VARIATION, match_curated=False,
-                                                                     exclude_curated=True)
-        self.assertTrue("ez72syb1455" in variations)
-
-    def test_is_variation_suspicious(self):
-        variations = self.ntt_extractor.extract_all_entities_by_type(self.text, EntityType.VARIATION,
-                                                                     match_curated=False,
-                                                                     exclude_curated=True)
-        is_susp = is_variation_suspicious(variations[0])
-        self.assertEqual(is_susp, "potentially new")
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/test_literature_index.py

@@ -1,29 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_tpc_config
-from wbtools.lib.nlp.literature_index.textpresso import TextpressoLiteratureIndex
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "textpresso.cfg")),
-                 "Textpresso test config file not present")
-class TestTextpressoLiteratureIndex(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_tpc_config()
-        self.lit_index = TextpressoLiteratureIndex(api_url=config["tpc"]["api_base_url"],
-                                                   api_token=config["tpc"]["api_token"], use_cache=True,
-                                                   corpora=[config["tpc"]["corpora"]])
-
-    def test_num_documents(self):
-        self.assertGreater(self.lit_index.num_documents(), 0)
-        self.assertGreater(self.lit_index.num_documents(), 0)
-
-    def test_count_matching_documents(self):
-        count = self.lit_index.count_matching_documents("DREAM complex")
-        self.assertGreater(count, 0)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/test_scraping.py

@@ -1,18 +0,0 @@
-import unittest
-
-from wbtools.lib.scraping import get_cgc_allele_designations, get_cgc_lab_designations
-
-
-class TestScraping(unittest.TestCase):
-
-    def test_get_cgc_allele_designations(self):
-        cgc_designations = get_cgc_allele_designations()
-        self.assertTrue(len(cgc_designations) > 0)
-
-    def test_get_cgc_lab_designations(self):
-        cgc_lab_designations = get_cgc_lab_designations()
-        self.assertTrue(len(cgc_lab_designations) > 0)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/test_text_preprocessing.py

@@ -1,59 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.lib.nlp.common import PaperSections
-from wbtools.lib.nlp.text_preprocessing import remove_sections_from_text, get_documents_from_text, preprocess
-from wbtools.literature.corpus import CorpusManager
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), '../data')
-
-
-class TestTextPreprocessing(unittest.TestCase):
-
-    def setUp(self):
-        config = read_db_config()
-        load_env_file()
-        self.cm = CorpusManager()
-        self.cm.load_from_wb_database(db_name=config["wb_database"]["db_name"],
-                                      db_user=config["wb_database"]["db_user"],
-                                      db_password=config["wb_database"]["db_password"],
-                                      db_host=config["wb_database"]["db_host"],
-                                      paper_ids=['00062455'])
-
-    def test_sectioning(self):
-        text = list(self.cm.corpus.values())[0].get_text_docs()[0]
-        self.assertTrue("REFERENCES" not in remove_sections_from_text(
-            text=text, sections_to_remove=[PaperSections.INTRODUCTION, PaperSections.REFERENCES],
-            must_be_present=[PaperSections.RESULTS]))
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_sectioning_cell_template(self):
-        config = read_db_config()
-        load_env_file()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=config["wb_database"]["db_name"], db_user=config["wb_database"]["db_user"],
-                                 db_password=config["wb_database"]["db_password"],
-                                 db_host=config["wb_database"]["db_host"],
-                                 paper_ids=['00059375'])
-        fulltext = cm.get_paper('00059375').get_text_docs(remove_sections=[PaperSections.REFERENCES],
-                                                          must_be_present=[PaperSections.METHOD, PaperSections.RESULTS])
-        self.assertTrue(fulltext)
-
-    def test_get_documents_from_text(self):
-        docs = get_documents_from_text(list(self.cm.corpus.values())[0].main_text)
-        self.assertEqual(len(docs), 1)
-        docs = get_documents_from_text(list(self.cm.corpus.values())[0].main_text, split_sentences=True)
-        self.assertGreater(len(docs), 10)
-
-    def test_preprocess(self):
-        doc = get_documents_from_text(list(self.cm.corpus.values())[0].main_text)[0]
-        prep_text = preprocess(doc, lower=True, tokenize=True, remove_alpha=True, remove_stopwords=True)
-        self.assertGreater(len(prep_text), 10)
-        self.assertTrue(all([k == k.lower() for k in prep_text]))
-        self.assertTrue("the" not in prep_text)
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/lib/test_text_similarity.py

@@ -1,52 +0,0 @@
-import os
-import unittest
-
-from gensim.models import Word2Vec
-from gensim.test.utils import common_texts
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.lib.nlp.common import PaperSections
-from wbtools.lib.nlp.text_similarity import get_softcosine_index, get_similar_documents
-from wbtools.literature.corpus import CorpusManager
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), '../data')
-
-
-class TestTextSimilarity(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        load_env_file()
-        self.cm = CorpusManager()
-        self.cm.load_from_wb_database(db_name=config["wb_database"]["db_name"],
-                                      db_user=config["wb_database"]["db_user"],
-                                      db_password=config["wb_database"]["db_password"],
-                                      db_host=config["wb_database"]["db_host"],
-                                      paper_ids=['00026804', '00026805', '00026806', '00026807', '00026808',
-                                                 '00026809', '00026810', '00026811', '00026812', '00026813',
-                                                 '00026814', '00026815', '00026816', '00026817', '00026818',
-                                                 '00026819', '00026820', '00026821', '00026822', '00026823'])
-
-    def test_soft_cosine_similarity(self):
-        corpus_list_token, idx_paperid_map = self.cm.get_flat_corpus_list_and_idx_paperid_map(
-            split_sentences=False, remove_sections=[PaperSections.REFERENCES], lowercase=True, tokenize=True,
-            remove_stopwords=True, remove_alpha=True)
-        model = Word2Vec(common_texts, min_count=1)
-        docsim_index, dictionary = get_softcosine_index(model=model, corpus_list_token=corpus_list_token)
-        query_docs = [corpus_list_token[0], corpus_list_token[1]]
-        sim_docs = get_similar_documents(docsim_index, dictionary, query_docs, idx_paperid_map)
-        self.assertTrue(idx_paperid_map[0] in [sim_doc.paper_id for sim_doc in sim_docs[0:2]])
-        self.assertTrue(idx_paperid_map[1] in [sim_doc.paper_id for sim_doc in sim_docs[0:2]])
-
-        corpus_list_token, idx_paperid_map = self.cm.get_flat_corpus_list_and_idx_paperid_map(
-            split_sentences=True, remove_sections=[PaperSections.REFERENCES], lowercase=True, tokenize=True,
-            remove_stopwords=True, remove_alpha=True)
-        docsim_index, dictionary = get_softcosine_index(model=model, corpus_list_token=corpus_list_token)
-        query_docs = [corpus_list_token[0], corpus_list_token[1]]
-        sim_docs = get_similar_documents(docsim_index, dictionary, query_docs, idx_paperid_map)
-        self.assertTrue(idx_paperid_map[0] in [sim_doc.paper_id for sim_doc in sim_docs[0:2]])
-        self.assertTrue(idx_paperid_map[1] in [sim_doc.paper_id for sim_doc in sim_docs[0:2]])
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/literature/test_corpus.py

@@ -1,86 +0,0 @@
-import os
-import unittest
-import tempfile
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.lib.nlp.common import PaperSections
-from wbtools.literature.corpus import CorpusManager
-from wbtools.literature.paper import WBPaper
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), '../data', 'text_files')
-
-
-@unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                 "local_config", "db.cfg")), "Test DB config file not present")
-class TestCorpusManager(unittest.TestCase):
-
-    def setUp(self) -> None:
-        config = read_db_config()
-        load_env_file()
-        self.cm = CorpusManager()
-        self.cm.load_from_wb_database(db_host=config["wb_database"]["db_host"],
-                                      db_user=config["wb_database"]["db_user"],
-                                      db_password=config["wb_database"]["db_password"],
-                                      db_name=config["wb_database"]["db_name"], max_num_papers=2)
-
-    def test_get_flat_corpus_list_and_idx_paperid_map(self):
-        flat_list, idx_paperid_map = self.cm.get_flat_corpus_list_and_idx_paperid_map(
-            tokenize=True, remove_stopwords=True, remove_alpha=True)
-        self.assertEqual(len(flat_list), len(idx_paperid_map))
-        self.assertTrue(len(idx_paperid_map[0]) == 8)
-        self.assertEqual(len(set(idx_paperid_map.values())), self.cm.size())
-
-    def test_split_sentences(self):
-        cl1, map1 = self.cm.get_flat_corpus_list_and_idx_paperid_map(tokenize=False, remove_stopwords=False,
-                                                                     remove_alpha=False, split_sentences=True)
-        cl2, map2 = self.cm.get_flat_corpus_list_and_idx_paperid_map(tokenize=True, remove_stopwords=True,
-                                                                     remove_alpha=True, split_sentences=True)
-        self.assertEqual(len(cl1), len(cl2))
-
-    def test_exclude_sections(self):
-        test_list, _ = self.cm.get_flat_corpus_list_and_idx_paperid_map(
-            split_sentences=False, remove_sections=[PaperSections.INTRODUCTION, PaperSections.REFERENCES],
-            must_be_present=[PaperSections.RESULTS], lowercase=False, tokenize=False, remove_stopwords=False,
-            remove_alpha=False)
-        pass
-
-    def test_load_from_wb_database(self):
-        db_config = read_db_config()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=db_config["wb_database"]["db_name"], db_user=db_config["wb_database"]["db_user"],
-                                 db_password=db_config["wb_database"]["db_password"],
-                                 db_host=db_config["wb_database"]["db_host"], max_num_papers=2)
-        self.assertTrue(cm.size() == 2)
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=db_config["wb_database"]["db_name"],
-                                 db_user=db_config["wb_database"]["db_user"],
-                                 db_password=db_config["wb_database"]["db_password"],
-                                 db_host=db_config["wb_database"]["db_host"], max_num_papers=2,
-                                 exclude_temp_pdf=True)
-        self.assertFalse(any([paper.is_temp() for paper in cm.get_all_papers()]))
-
-    def test_load_supplemental(self):
-        db_config = read_db_config()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_name=db_config["wb_database"]["db_name"],
-                                 db_user=db_config["wb_database"]["db_user"],
-                                 db_password=db_config["wb_database"]["db_password"],
-                                 db_host=db_config["wb_database"]["db_host"],
-                                 paper_ids=["00062512"])
-        self.assertTrue(len(cm.get_paper("00062512").supplemental_docs) > 0)
-
-    # def test_load_from_wb_database_afp(self):
-    #     db_config = read_db_config()
-    #     cm = CorpusManager()
-    #     cm.load_from_wb_database(db_name=db_config["wb_database"]["db_name"],
-    #                              db_user=db_config["wb_database"]["db_user"],
-    #                              db_password=db_config["wb_database"]["db_password"],
-    #                              db_host=db_config["wb_database"]["db_host"],
-    #                              max_num_papers=2,
-    #                              load_curation_info=True, load_afp_info=True,
-    #                              exclude_temp_pdf=True, exclude_afp_processed=True, must_be_autclass_flagged=True)
-    #     self.assertFalse(any([paper.afp_processed for paper in cm.get_all_papers()]))
-
-
-if __name__ == '__main__':
-    unittest.main()

wbtools-3.1.0/tests/literature/test_paper.py

@@ -1,110 +0,0 @@
-import os
-import unittest
-
-from tests.config_reader import read_db_config, load_env_file
-from wbtools.db.paper import WBPaperDBManager
-from wbtools.lib.nlp.common import PaperSections
-from wbtools.literature.corpus import CorpusManager
-from wbtools.literature.paper import WBPaper
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), '../data', 'text_files')
-
-
-class TestWBPaper(unittest.TestCase):
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_load_curation_info_from_db(self):
-        config = read_db_config()
-        load_env_file()
-        paper = WBPaper(paper_id="00004161", db_manager=WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"]))
-        paper.paper_id = "00004161"
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        paper.load_curation_info_from_db()
-        self.assertTrue(paper.aut_class_values["seqchange"] == 'high')
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_load_bib_info(self):
-        config = read_db_config()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00004161", db_manager=db_manager)
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        with db_manager:
-            paper.load_bib_info()
-        self.assertGreater(len(paper.authors), 0)
-        self.assertTrue(paper.abstract.startswith("Niemann-Pick type C (NP-C) disease is a progressive"))
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_extract_all_email_addresses_from_text(self):
-        config = read_db_config()
-        load_env_file()
-        cm = CorpusManager()
-        cm.load_from_wb_database(db_host=config["wb_database"]["db_host"],
-                                 db_user=config["wb_database"]["db_user"],
-                                 db_password=config["wb_database"]["db_password"],
-                                 db_name=config["wb_database"]["db_name"], paper_ids=['00026804'])
-        addresses = cm.get_paper('00026804').extract_all_email_addresses_from_text()
-        self.assertGreater(len(addresses), 0)
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_pdf2txt_conversion(self):
-        config = read_db_config()
-        load_env_file()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00003969", db_manager=db_manager)
-        paper.load_bib_info()
-        paper.load_text_from_pdf_files()
-        fulltext = paper.get_text_docs()
-        self.assertGreater(len(fulltext), 0)
-        self.assertTrue("u253, u423 deletion in exon 3" in fulltext[0])
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_pdf_table_conversion(self):
-        config = read_db_config()
-        load_env_file()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00059755", db_manager=db_manager)
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        paper.load_text_from_pdf_files()
-        fulltext = paper.get_text_docs()
-        self.assertTrue(fulltext)
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_tokenize_sentences_with_tables(self):
-        config = read_db_config()
-        load_env_file()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00003969", db_manager=db_manager)
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        paper.load_text_from_pdf_files()
-        sentences = paper.get_text_docs(split_sentences=True)
-        self.assertGreater(len(sentences), 0)
-
-    @unittest.skipIf(not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data",
-                                                     "local_config", "db.cfg")), "Test DB config file not present")
-    def test_two_cols_conversion(self):
-        config = read_db_config()
-        load_env_file()
-        db_manager = WBPaperDBManager(
-            dbname=config["wb_database"]["db_name"], user=config["wb_database"]["db_user"],
-            password=config["wb_database"]["db_password"], host=config["wb_database"]["db_host"])
-        paper = WBPaper(paper_id="00055367", db_manager=db_manager)
-        paper.agr_curie = paper.db_manager.get_paper_curie(paper.paper_id)
-        paper.load_text_from_pdf_files()
-        fulltext = paper.get_text_docs()
-        self.assertTrue(fulltext)