wbtools 3.3.0__tar.gz → 3.5.0__tar.gz

{wbtools-3.3.0/wbtools.egg-info → wbtools-3.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wbtools
-Version: 3.3.0
+Version: 3.5.0
 Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
 Author-email: Valerio Arnaboldi <valearna@caltech.edu>
 Project-URL: Homepage, https://github.com/WormBase/wbtools
@@ -25,6 +25,7 @@ Requires-Dist: python-dateutil~=2.8.2
 Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: grobid-client
 Requires-Dist: agr-cognito-py
+Requires-Dist: agr-abc-document-parsers>=1.7.2
 
 # WBtools
 > Interface to WormBase curation database and Text Mining functions

{wbtools-3.3.0 → wbtools-3.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "wbtools"
-version = "3.3.0"
+version = "3.5.0"
 authors = [
     {name = "Valerio Arnaboldi", email = "valearna@caltech.edu"},
 ]
@@ -31,6 +31,7 @@ dependencies = [
     "python-dotenv~=1.0.0",
     "grobid-client",
     "agr-cognito-py",
+    "agr-abc-document-parsers>=1.7.2",
 ]
 
 [project.urls]

wbtools-3.5.0/wbtools/literature/abc_search.py

@@ -0,0 +1,69 @@
+import logging
+from typing import Dict, Iterator, List, Tuple
+
+import requests
+
+from wbtools.literature.paper import ABC_API, ABCRequestError
+from wbtools.utils.auth_utils import get_authentication_token, generate_headers
+
+logger = logging.getLogger(__name__)
+
+SEARCH_PAGE_SIZE = 100
+WB_PAPER_XREF_PREFIX = "WB:WBPaper"
+
+
+def get_wb_paper_ids_from_abc(required_workflow_tags: Dict[str, List[str]], date_created_from: str,
+                              date_created_to: str) -> Iterator[Tuple[str, str]]:
+    """get the WB papers in the ABC corpus created in the given window that have all the required workflow tags,
+    newest first
+
+    Args:
+        required_workflow_tags (Dict[str, List[str]]): ABC search facet name (e.g. "file_workflow") -> ATP ids that
+                                                       must all be present on the paper
+        date_created_from (str): first creation date to include, YYYY-MM-DD
+        date_created_to (str): last creation date to include, YYYY-MM-DD
+
+    Returns:
+        Iterator[Tuple[str, str]]: WBPaper id (without the WBPaper prefix) and AGRKB curie of each paper
+
+    Raises:
+        ABCRequestError: if the ABC search fails
+    """
+    facets_values = {"mods_in_corpus.keyword": ["WB"]}
+    facets_values.update(required_workflow_tags)
+    page = 1
+    while True:
+        hits = _search_references({"facets_values": facets_values,
+                                   "date_created": [date_created_from, date_created_to],
+                                   "sort": [{"date_created": {"order": "desc"}}],
+                                   "size_result_count": SEARCH_PAGE_SIZE, "page": page})
+        for hit in hits:
+            wb_paper_id = _get_wb_paper_id(hit)
+            if wb_paper_id is None:
+                logger.warning(f"Skipping ABC reference {hit.get('curie')}: no WBPaper cross-reference")
+                continue
+            yield wb_paper_id, hit["curie"]
+        if len(hits) < SEARCH_PAGE_SIZE:
+            return
+        page += 1
+
+
+def _search_references(body: dict) -> list:
+    headers = generate_headers(get_authentication_token())
+    try:
+        response = requests.post(f"https://{ABC_API}/search/references/", json=body, headers=headers, timeout=300)
+        response.raise_for_status()
+        result = response.json()
+    except (requests.exceptions.RequestException, ValueError) as e:
+        raise ABCRequestError(f"ABC search failed: {e}") from e
+    if not isinstance(result, dict) or result.get("error") or "hits" not in result:
+        raise ABCRequestError(f"ABC search failed: {result}")
+    return result["hits"]
+
+
+def _get_wb_paper_id(hit: dict):
+    for xref in hit.get("cross_references") or []:
+        if xref.get("curie", "").startswith(WB_PAPER_XREF_PREFIX) and \
+                str(xref.get("is_obsolete")).lower() != "true":
+            return xref["curie"][len(WB_PAPER_XREF_PREFIX):]
+    return None

{wbtools-3.3.0 → wbtools-3.5.0}/wbtools/literature/corpus.py

@@ -45,7 +45,8 @@ class CorpusManager(object):
                               pap_types: List[str] = None,
                               exclude_afp_processed: bool = False, exclude_afp_not_curatable: bool = False,
                               exclude_no_main_text: bool = False, exclude_no_author_email: bool = False,
-                              main_file_only: bool = False) -> None:
+                              main_file_only: bool = False, text_source: str = "pdf",
+                              agr_curies: Dict[str, str] = None) -> None:
         """load papers from WormBase database
 
         Args:
@@ -69,7 +70,14 @@ class CorpusManager(object):
             exclude_afp_not_curatable (bool): whether to exclude papers that are not relevant for AFP curation
             exclude_no_main_text (bool): whether to exclude papers without a fulltext that can be converted to txt
             exclude_no_author_email (bool): whether to exclude papers without any contact email in WB
+            text_source (str): where to read the text of the papers from when load_pdf_files is True: "pdf" to
+                               convert the PDF files with GROBID, "abc_markdown" to read the Markdown files
+                               converted by the ABC
+            agr_curies (Dict[str, str]): AGRKB curies of the papers by paper id, e.g. from an ABC search. They take
+                                         precedence over the curies stored in the WB database
         """
+        if text_source not in ("pdf", "abc_markdown"):
+            raise ValueError(f"Unknown text_source {text_source}, use 'pdf' or 'abc_markdown'")
         main_db_manager = WBDBManager(db_name, db_user, db_password, db_host)
         with main_db_manager:
             if not paper_ids:
@@ -96,8 +104,9 @@ class CorpusManager(object):
 
         for paper_id in paper_ids:
             paper = WBPaper(paper_id=paper_id, db_manager=main_db_manager.paper)
-            paper.agr_curie = main_db_manager.paper.get_paper_curie(paper_id)
+            paper.agr_curie = (agr_curies or {}).get(paper_id) or main_db_manager.paper.get_paper_curie(paper_id)
             if paper.agr_curie is None:
+                logger.warning(f"Skipping paper {paper_id}: no AGRKB curie")
                 continue
             if exclude_afp_processed and paper_id in afp_processed_ids:
                 logger.info("Skipping paper already processed by AFP")
@@ -119,7 +128,8 @@ class CorpusManager(object):
                         continue
                     if exclude_no_author_email and not paper.get_authors_with_email_address_in_wb(
                             blacklisted_email_addresses=blacklisted_email_addresses):
-                        logger.info("Skipping paper without any email address in text with records in WB")
+                        logger.info("Skipping paper without any email address in ABC or WB authors with records "
+                                    "in WB")
                         continue
                 if load_afp_info:
                     logger.info("Loading AFP info for paper")
@@ -127,9 +137,13 @@ class CorpusManager(object):
                                                 paper_ids_full_submission=afp_full_submission_ids,
                                                 paper_ids_partial_submission=afp_partial_submission_ids)
             if load_pdf_files:
-                logger.info("Loading text from PDF files for paper")
-                if paper.load_text_from_pdf_files(main_file_only=main_file_only) is False:
-                    continue
+                if text_source == "abc_markdown":
+                    logger.info("Loading text from ABC Markdown files for paper")
+                    paper.load_text_from_abc_markdown()
+                else:
+                    logger.info("Loading text from PDF files for paper")
+                    if paper.load_text_from_pdf_files(main_file_only=main_file_only) is False:
+                        continue
                 if exclude_temp_pdf and paper.is_temp():
                     logger.info("Skipping proof paper")
                     continue

{wbtools-3.3.0 → wbtools-3.5.0}/wbtools/literature/paper.py

@@ -17,6 +17,7 @@ from pathlib import Path
 from grobid_client.api.pdf import process_fulltext_document
 from grobid_client.models import Article, ProcessForm
 from grobid_client.types import TEI, File
+from agr_abc_document_parsers import extract_sentences, read_markdown
 
 
 from wbtools.db.afp import WBAFPDBManager
@@ -69,6 +70,10 @@ def convert_pdf_to_txt(file_path):
         return []
 
 
+class ABCRequestError(Exception):
+    """raised when data that is required to process a paper cannot be retrieved from the ABC"""
+
+
 def get_data_from_url(url, headers=None, file_type='json'):
     try:
         response = requests.request("GET", url, headers=headers)
@@ -85,6 +90,22 @@ def get_data_from_url(url, headers=None, file_type='json'):
         return None
 
 
+MARKDOWN_MAIN_FILE_CLASS = "converted_merged_main"
+MARKDOWN_SUPPLEMENT_FILE_CLASS = "converted_merged_supplement"
+
+
+def _decode_markdown(content: bytes) -> str:
+    try:
+        return content.decode("utf-8")
+    except UnicodeDecodeError:
+        return content.decode("latin-1", errors="replace")
+
+
+def _is_wb_or_shared_file(ref_file: dict) -> bool:
+    mods = ref_file.get("referencefile_mods") or []
+    return not mods or any(mod.get("mod_abbreviation") in (None, "WB") for mod in mods)
+
+
 class WBPaper(object):
     """WormBase paper information"""
 
@@ -115,6 +136,7 @@ class WBPaper(object):
         self.afp_processed = False
         self.afp_partial_submission = False
         self.afp_contact_emails = []
+        self.abc_email_addresses = None
         self.db_manager = db_manager
 
     def get_corresponding_author(self) -> Union[WBAuthor, None]:
@@ -213,6 +235,56 @@ class WBPaper(object):
                 return False
             return added_ref_files > 0
 
+    def load_text_from_abc_markdown(self) -> bool:
+        """load the main text and the supplements of the paper from the Markdown files converted by the ABC
+
+        Returns:
+            bool: True, the main text has been loaded
+
+        Raises:
+            ABCRequestError: if the list of files, the main Markdown file or its conversion to sentences fails
+        """
+        headers = generate_headers(get_authentication_token())
+        ref_files = get_data_from_url(f"https://{ABC_API}/reference/referencefile/show_all/{self.agr_curie}", headers)
+        if ref_files is None:
+            raise ABCRequestError(f"Could not list the ABC files of paper {self.paper_id} ({self.agr_curie})")
+        markdown_files = [ref_file for ref_file in ref_files if ref_file.get("file_extension") == "md"]
+        main_files = [ref_file for ref_file in markdown_files
+                      if ref_file.get("file_class") == MARKDOWN_MAIN_FILE_CLASS]
+        if not main_files:
+            raise ABCRequestError(f"No converted Markdown file in ABC for paper {self.paper_id} ({self.agr_curie})")
+        main_file = next((ref_file for ref_file in main_files if _is_wb_or_shared_file(ref_file)), main_files[0])
+        try:
+            main_text = self._get_sentences_from_abc_markdown(main_file, headers)
+        except Exception as e:
+            raise ABCRequestError(f"Could not load the converted Markdown file of paper {self.paper_id} "
+                                  f"({self.agr_curie}): {e}") from e
+        if not main_text:
+            raise ABCRequestError(f"The converted Markdown file of paper {self.paper_id} ({self.agr_curie}) "
+                                  f"has no text")
+        self.main_text = main_text
+        for supplement in markdown_files:
+            if supplement.get("file_class") != MARKDOWN_SUPPLEMENT_FILE_CLASS or \
+                    not _is_wb_or_shared_file(supplement):
+                continue
+            try:
+                supplement_text = self._get_sentences_from_abc_markdown(supplement, headers)
+            except Exception as e:
+                logger.warning(f"Skipping supplement {supplement.get('display_name')} of paper {self.paper_id}: {e}")
+                continue
+            if supplement_text:
+                self.supplemental_docs.append(supplement_text)
+        return True
+
+    @staticmethod
+    def _get_sentences_from_abc_markdown(ref_file: dict, headers: dict) -> List[str]:
+        # file_type='pdf' makes get_data_from_url return the raw bytes of the file
+        content = get_data_from_url(f"https://{ABC_API}/reference/referencefile/download_file/"
+                                    f"{ref_file['referencefile_id']}", headers, file_type='pdf')
+        if not content:
+            raise ValueError(f"download of ABC file {ref_file['referencefile_id']} failed")
+        return extract_sentences(read_markdown(_decode_markdown(content)))
+
     def load_curation_info_from_db(self):
         """load curation data from WormBase database"""
         if self.db_manager:
@@ -322,46 +394,64 @@ class WBPaper(object):
     def extract_all_email_addresses_from_text_and_write_to_db(self):
         self.write_email_addresses_to_db(self.extract_all_email_addresses_from_text())
 
+    def get_abc_email_addresses(self) -> Union[List[str], None]:
+        """get the email addresses associated with the paper in the ABC
+
+        Returns:
+            Union[List[str], None]: the list of email addresses, or None if the ABC request failed
+        """
+        if self.abc_email_addresses is None:
+            reference_emails_api = f"https://{ABC_API}/reference/{self.agr_curie}/emails"
+            headers = generate_headers(get_authentication_token())
+            reference_emails = get_data_from_url(reference_emails_api, headers)
+            if reference_emails is not None:
+                self.abc_email_addresses = [reference_email["email_address"] for reference_email in
+                                            reference_emails]
+        return self.abc_email_addresses
+
     def get_aut_class_value_for_datatype(self, datatype: str):
         return self.aut_class_values[datatype] if self.aut_class_values[datatype] else None
 
     def get_authors_with_email_address_in_wb(self, blacklisted_email_addresses: List[str] = None,
                                              first_only: bool = False) -> Union[List[Tuple[WBAuthor, str]], None]:
         """
-        Get the first email address in the paper with a corresponding person entry in WB and return the person object
-        and the email address found in the paper, which may be more recent than the one in WB
+        Get the email addresses associated with the paper in the ABC that have a corresponding person entry in WB and
+        return the person object and the email address from the ABC, which may be more recent than the one in WB
 
         Args:
             blacklisted_email_addresses (List[str]): a list of email addresses to be excluded from the search
             first_only (bool): whether to return only the first available author
 
         Returns:
-            Union[List[Tuple[WBPerson, str]], None]: a tuple containing the WBPerson and the email address found in the paper.
+            Union[List[Tuple[WBPerson, str]], None]: a tuple containing the WBPerson and the email address from the ABC.
                                                If no email is found with a corresponding person in WB, then the function
                                                will return the corresponding author associated with the paper in WB and
                                                its email address, if any, otherwise None.
+
+        Raises:
+            ABCRequestError: if the email addresses cannot be retrieved from the ABC
         """
         result = []
-        extracted_addresses = self.extract_all_email_addresses_from_text()
-        if not extracted_addresses:
-            extracted_addresses = self.extract_all_email_addresses_from_text(self.get_text_docs(
-                include_supplemental=False, return_concatenated=True).replace(". ", "."))
-        blacklisted_email_addresses = set(blacklisted_email_addresses)
-        if extracted_addresses:
-            for extracted_address in extracted_addresses:
-                if "'" not in extracted_address:
+        abc_addresses = self.get_abc_email_addresses()
+        if abc_addresses is None:
+            raise ABCRequestError(f"Could not retrieve email addresses from ABC for paper {self.paper_id} "
+                                  f"({self.agr_curie})")
+        blacklisted_email_addresses = set(blacklisted_email_addresses or [])
+        if abc_addresses:
+            for abc_address in abc_addresses:
+                if "'" not in abc_address:
                     person_id = self.db_manager.get_db_manager(
-                        WBPersonDBManager).get_person_id_from_email_address(extracted_address)
+                        WBPersonDBManager).get_person_id_from_email_address(abc_address)
                     if person_id:
                         current_address = self.db_manager.get_db_manager(
                                 WBPersonDBManager).get_current_email_address_for_person(person_id)
                         if current_address and current_address not in blacklisted_email_addresses:
                             result.append((self.db_manager.get_db_manager(WBPersonDBManager).get_person(
                                 person_id=person_id), current_address))
-                        if (extracted_address not in blacklisted_email_addresses and
-                                extracted_address != current_address):
+                        if (abc_address not in blacklisted_email_addresses and
+                                abc_address != current_address):
                             result.append((self.db_manager.get_db_manager(WBPersonDBManager).get_person(
-                                person_id=person_id), extracted_address))
+                                person_id=person_id), abc_address))
                         if first_only and result:
                             return result
         if not result:

{wbtools-3.3.0 → wbtools-3.5.0/wbtools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wbtools
-Version: 3.3.0
+Version: 3.5.0
 Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
 Author-email: Valerio Arnaboldi <valearna@caltech.edu>
 Project-URL: Homepage, https://github.com/WormBase/wbtools
@@ -25,6 +25,7 @@ Requires-Dist: python-dateutil~=2.8.2
 Requires-Dist: python-dotenv~=1.0.0
 Requires-Dist: grobid-client
 Requires-Dist: agr-cognito-py
+Requires-Dist: agr-abc-document-parsers>=1.7.2
 
 # WBtools
 > Interface to WormBase curation database and Text Mining functions

{wbtools-3.3.0 → wbtools-3.5.0}/wbtools.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ wbtools/lib/nlp/literature_index/__init__.py
 wbtools/lib/nlp/literature_index/abstract_index.py
 wbtools/lib/nlp/literature_index/textpresso.py
 wbtools/literature/__init__.py
+wbtools/literature/abc_search.py
 wbtools/literature/corpus.py
 wbtools/literature/paper.py
 wbtools/literature/person.py

{wbtools-3.3.0 → wbtools-3.5.0}/wbtools.egg-info/requires.txt

@@ -12,3 +12,4 @@ python-dateutil~=2.8.2
 python-dotenv~=1.0.0
 grobid-client
 agr-cognito-py
+agr-abc-document-parsers>=1.7.2