wbtools 3.2.0__tar.gz → 3.4.0__tar.gz

{wbtools-3.2.0/wbtools.egg-info → wbtools-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wbtools
-Version: 3.2.0
+Version: 3.4.0
 Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
 Author-email: Valerio Arnaboldi <valearna@caltech.edu>
 Project-URL: Homepage, https://github.com/WormBase/wbtools
@@ -24,6 +24,7 @@ Requires-Dist: requests~=2.31.0
 Requires-Dist: python-dateutil~=2.8.2
 Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: grobid-client
+Requires-Dist: agr-cognito-py
 
 # WBtools
 > Interface to WormBase curation database and Text Mining functions

{wbtools-3.2.0 → wbtools-3.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "wbtools"
-version = "3.2.0"
+version = "3.4.0"
 authors = [
     {name = "Valerio Arnaboldi", email = "valearna@caltech.edu"},
 ]
@@ -30,6 +30,7 @@ dependencies = [
     "python-dateutil~=2.8.2",
     "python-dotenv~=1.0.0",
     "grobid-client",
+    "agr-cognito-py",
 ]
 
 [project.urls]

{wbtools-3.2.0 → wbtools-3.4.0}/wbtools/literature/corpus.py

@@ -8,7 +8,7 @@ from wbtools.db.dbmanager import WBDBManager
 from wbtools.lib.nlp.common import PaperSections
 from wbtools.lib.nlp.text_preprocessing import preprocess
 from wbtools.lib.nlp.text_similarity import get_softcosine_index, get_similar_documents, SimilarityResult
-from wbtools.literature.paper import WBPaper
+from wbtools.literature.paper import WBPaper, ABCRequestError
 
 
 logger = logging.getLogger(__name__)
@@ -117,10 +117,17 @@ class CorpusManager(object):
                     logger.info("Loading bib info for paper")
                     if paper.load_bib_info() is False:
                         continue
-                    if exclude_no_author_email and not paper.get_authors_with_email_address_in_wb(
-                            blacklisted_email_addresses=blacklisted_email_addresses):
-                        logger.info("Skipping paper without any email address in text with records in WB")
-                        continue
+                    if exclude_no_author_email:
+                        try:
+                            authors_with_email = paper.get_authors_with_email_address_in_wb(
+                                blacklisted_email_addresses=blacklisted_email_addresses)
+                        except ABCRequestError as e:
+                            logger.warning(f"Skipping paper: {e}")
+                            continue
+                        if not authors_with_email:
+                            logger.info("Skipping paper without any email address in ABC or WB authors with records "
+                                        "in WB")
+                            continue
                 if load_afp_info:
                     logger.info("Loading AFP info for paper")
                     paper.load_afp_info_from_db(paper_ids_no_submission=afp_no_submission_ids,

{wbtools-3.2.0 → wbtools-3.4.0}/wbtools/literature/paper.py

@@ -26,7 +26,7 @@ from wbtools.lib.nlp.entity_extraction.email_addresses import get_email_addresse
 from wbtools.lib.nlp.text_preprocessing import preprocess, get_documents_from_text, PaperSections
 from wbtools.lib.timeout import timeout
 from wbtools.literature.person import WBAuthor
-from wbtools.utils.okta_utils import get_authentication_token, generate_headers
+from wbtools.utils.auth_utils import get_authentication_token, generate_headers
 
 logger = logging.getLogger(__name__)
 
@@ -69,6 +69,10 @@ def convert_pdf_to_txt(file_path):
         return []
 
 
+class ABCRequestError(Exception):
+    """raised when data that is required to process a paper cannot be retrieved from the ABC"""
+
+
 def get_data_from_url(url, headers=None, file_type='json'):
     try:
         response = requests.request("GET", url, headers=headers)
@@ -115,6 +119,7 @@ class WBPaper(object):
         self.afp_processed = False
         self.afp_partial_submission = False
         self.afp_contact_emails = []
+        self.abc_email_addresses = None
         self.db_manager = db_manager
 
     def get_corresponding_author(self) -> Union[WBAuthor, None]:
@@ -193,6 +198,7 @@ class WBPaper(object):
             blue_api_base_url = os.environ.get('API_SERVER', "literature-rest.alliancegenome.org")
             all_reffiles_for_pap_api = f'https://{blue_api_base_url}/reference/referencefile/show_all/{self.agr_curie}'
             request = urllib.request.Request(url=all_reffiles_for_pap_api)
+            request.add_header("Authorization", f"Bearer {get_authentication_token()}")
             request.add_header("Content-type", "application/json")
             request.add_header("Accept", "application/json")
             added_ref_files = 0
@@ -227,7 +233,9 @@ class WBPaper(object):
             raise Exception("PaperDBManager not set")
         # Get Alliance reference info from WBPaperID
         ref_info_from_xref_api = f"https://{ABC_API}/reference/by_cross_reference/WB:WBPaper{self.paper_id}"
-        ref_info: dict = get_data_from_url(ref_info_from_xref_api)
+        token = get_authentication_token()
+        headers = generate_headers(token)
+        ref_info: dict = get_data_from_url(ref_info_from_xref_api, headers)
         if ref_info:
             self.abstract = ref_info["abstract"]
             self.title = ref_info["title"]
@@ -319,46 +327,64 @@ class WBPaper(object):
     def extract_all_email_addresses_from_text_and_write_to_db(self):
         self.write_email_addresses_to_db(self.extract_all_email_addresses_from_text())
 
+    def get_abc_email_addresses(self) -> Union[List[str], None]:
+        """get the email addresses associated with the paper in the ABC
+
+        Returns:
+            Union[List[str], None]: the list of email addresses, or None if the ABC request failed
+        """
+        if self.abc_email_addresses is None:
+            reference_emails_api = f"https://{ABC_API}/reference/{self.agr_curie}/emails"
+            headers = generate_headers(get_authentication_token())
+            reference_emails = get_data_from_url(reference_emails_api, headers)
+            if reference_emails is not None:
+                self.abc_email_addresses = [reference_email["email_address"] for reference_email in
+                                            reference_emails]
+        return self.abc_email_addresses
+
     def get_aut_class_value_for_datatype(self, datatype: str):
         return self.aut_class_values[datatype] if self.aut_class_values[datatype] else None
 
     def get_authors_with_email_address_in_wb(self, blacklisted_email_addresses: List[str] = None,
                                              first_only: bool = False) -> Union[List[Tuple[WBAuthor, str]], None]:
         """
-        Get the first email address in the paper with a corresponding person entry in WB and return the person object
-        and the email address found in the paper, which may be more recent than the one in WB
+        Get the email addresses associated with the paper in the ABC that have a corresponding person entry in WB and
+        return the person object and the email address from the ABC, which may be more recent than the one in WB
 
         Args:
             blacklisted_email_addresses (List[str]): a list of email addresses to be excluded from the search
             first_only (bool): whether to return only the first available author
 
         Returns:
-            Union[List[Tuple[WBPerson, str]], None]: a tuple containing the WBPerson and the email address found in the paper.
+            Union[List[Tuple[WBPerson, str]], None]: a tuple containing the WBPerson and the email address from the ABC.
                                                If no email is found with a corresponding person in WB, then the function
                                                will return the corresponding author associated with the paper in WB and
                                                its email address, if any, otherwise None.
+
+        Raises:
+            ABCRequestError: if the email addresses cannot be retrieved from the ABC
         """
         result = []
-        extracted_addresses = self.extract_all_email_addresses_from_text()
-        if not extracted_addresses:
-            extracted_addresses = self.extract_all_email_addresses_from_text(self.get_text_docs(
-                include_supplemental=False, return_concatenated=True).replace(". ", "."))
-        blacklisted_email_addresses = set(blacklisted_email_addresses)
-        if extracted_addresses:
-            for extracted_address in extracted_addresses:
-                if "'" not in extracted_address:
+        abc_addresses = self.get_abc_email_addresses()
+        if abc_addresses is None:
+            raise ABCRequestError(f"Could not retrieve email addresses from ABC for paper {self.paper_id} "
+                                  f"({self.agr_curie})")
+        blacklisted_email_addresses = set(blacklisted_email_addresses or [])
+        if abc_addresses:
+            for abc_address in abc_addresses:
+                if "'" not in abc_address:
                     person_id = self.db_manager.get_db_manager(
-                        WBPersonDBManager).get_person_id_from_email_address(extracted_address)
+                        WBPersonDBManager).get_person_id_from_email_address(abc_address)
                     if person_id:
                         current_address = self.db_manager.get_db_manager(
                                 WBPersonDBManager).get_current_email_address_for_person(person_id)
                         if current_address and current_address not in blacklisted_email_addresses:
                             result.append((self.db_manager.get_db_manager(WBPersonDBManager).get_person(
                                 person_id=person_id), current_address))
-                        if (extracted_address not in blacklisted_email_addresses and
-                                extracted_address != current_address):
+                        if (abc_address not in blacklisted_email_addresses and
+                                abc_address != current_address):
                             result.append((self.db_manager.get_db_manager(WBPersonDBManager).get_person(
-                                person_id=person_id), extracted_address))
+                                person_id=person_id), abc_address))
                         if first_only and result:
                             return result
         if not result:

{wbtools-3.2.0 → wbtools-3.4.0}/wbtools/utils/alliance.py

@@ -3,7 +3,7 @@ import logging
 import urllib.request
 
 from wbtools.lib.nlp.common import EntityType
-from wbtools.utils.okta_utils import get_authentication_token
+from wbtools.utils.auth_utils import get_authentication_token
 
 
 logger = logging.getLogger(__name__)

wbtools-3.4.0/wbtools/utils/auth_utils.py

@@ -0,0 +1,21 @@
+"""
+Authentication utilities for AGR API access.
+
+This module provides authentication token generation using AWS Cognito
+via the agr_cognito_py library. The functions are re-exported for
+backward compatibility with code that was using the old Okta-based
+authentication.
+
+Required environment variables for Cognito authentication:
+- COGNITO_ADMIN_CLIENT_ID
+- COGNITO_ADMIN_CLIENT_SECRET
+- COGNITO_TOKEN_URL
+- COGNITO_ADMIN_SCOPE
+"""
+
+from agr_cognito_py import (
+    get_admin_token as get_authentication_token,
+    generate_headers
+)
+
+__all__ = ['get_authentication_token', 'generate_headers']

{wbtools-3.2.0 → wbtools-3.4.0/wbtools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wbtools
-Version: 3.2.0
+Version: 3.4.0
 Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
 Author-email: Valerio Arnaboldi <valearna@caltech.edu>
 Project-URL: Homepage, https://github.com/WormBase/wbtools
@@ -24,6 +24,7 @@ Requires-Dist: requests~=2.31.0
 Requires-Dist: python-dateutil~=2.8.2
 Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: grobid-client
+Requires-Dist: agr-cognito-py
 
 # WBtools
 > Interface to WormBase curation database and Text Mining functions

{wbtools-3.2.0 → wbtools-3.4.0}/wbtools.egg-info/SOURCES.txt

@@ -39,4 +39,4 @@ wbtools/literature/paper.py
 wbtools/literature/person.py
 wbtools/utils/__init__.py
 wbtools/utils/alliance.py
-wbtools/utils/okta_utils.py
+wbtools/utils/auth_utils.py

{wbtools-3.2.0 → wbtools-3.4.0}/wbtools.egg-info/requires.txt

@@ -11,3 +11,4 @@ requests~=2.31.0
 python-dateutil~=2.8.2
 python-dotenv~=1.0.0
 grobid-client
+agr-cognito-py

wbtools-3.2.0/wbtools/utils/okta_utils.py

@@ -1,67 +0,0 @@
-import os
-from json import loads
-from os import environ, path
-import logging
-import logging.config
-from datetime import datetime
-from dateutil.relativedelta import relativedelta
-import requests
-
-
-def generate_headers(token):
-    """
-
-    :param token:
-    :return:
-    """
-
-    authorization = 'Bearer ' + token
-    headers = {
-        'Authorization': authorization,
-        'Content-Type': 'application/json',
-        'Accept': 'application/json'
-    }
-    return headers
-
-
-def update_okta_token():  # pragma: no cover
-    """
-
-    :return:
-    """
-
-    url = f"https://{os.environ.get('OKTA_DOMAIN')}/v1/token"
-    headers = {
-        'Content-Type': 'application/x-www-form-urlencoded',
-        'Accept': 'application/json'
-    }
-    data_dict = dict()
-    data_dict['grant_type'] = 'client_credentials'
-    data_dict['client_id'] = environ.get('OKTA_CLIENT_ID')
-    data_dict['client_secret'] = environ.get('OKTA_CLIENT_SECRET')
-    data_dict['scope'] = 'admin'
-    post_return = requests.post(url, headers=headers, data=data_dict)
-    logging.warning(post_return.text)
-    response_dict = loads(post_return.text)
-    token = response_dict['access_token']
-    okta_file = environ.get('TMP_PATH') + 'okta_token'
-    os.makedirs(environ.get('TMP_PATH'), exist_ok=True)
-    with open(okta_file, 'w') as okta_fh:
-        okta_fh.write("%s" % token)
-    return token
-
-
-def get_authentication_token():  # pragma: no cover
-    okta_file = environ.get('TMP_PATH') + 'okta_token'
-    token = ''
-    if path.isfile(okta_file):
-        one_day_ago = datetime.now() - relativedelta(days=1)
-        file_time = datetime.fromtimestamp(path.getmtime(okta_file))
-        if file_time > one_day_ago:
-            with open(okta_file, 'r') as okta_fh:
-                token = okta_fh.read().replace("\n", "")
-        else:
-            token = update_okta_token()
-    else:
-        token = update_okta_token()
-    return token