wbtools 3.7.0__tar.gz → 3.7.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {wbtools-3.7.0/wbtools.egg-info → wbtools-3.7.2}/PKG-INFO +1 -1
  2. {wbtools-3.7.0 → wbtools-3.7.2}/pyproject.toml +1 -1
  3. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/paper.py +13 -1
  4. {wbtools-3.7.0 → wbtools-3.7.2/wbtools.egg-info}/PKG-INFO +1 -1
  5. {wbtools-3.7.0 → wbtools-3.7.2}/LICENSE +0 -0
  6. {wbtools-3.7.0 → wbtools-3.7.2}/README.md +0 -0
  7. {wbtools-3.7.0 → wbtools-3.7.2}/setup.cfg +0 -0
  8. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/__init__.py +0 -0
  9. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/__init__.py +0 -0
  10. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/abstract_manager.py +0 -0
  11. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/afp.py +0 -0
  12. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/antibody.py +0 -0
  13. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/dbmanager.py +0 -0
  14. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/expression.py +0 -0
  15. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/gene.py +0 -0
  16. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/generic.py +0 -0
  17. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/paper.py +0 -0
  18. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/person.py +0 -0
  19. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/__init__.py +0 -0
  20. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/email/__init__.py +0 -0
  21. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/email/generic.py +0 -0
  22. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/__init__.py +0 -0
  23. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/common.py +0 -0
  24. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/__init__.py +0 -0
  25. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/email_addresses.py +0 -0
  26. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/ntt_extractor.py +0 -0
  27. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/variations.py +0 -0
  28. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/literature_index/__init__.py +0 -0
  29. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/literature_index/abstract_index.py +0 -0
  30. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/literature_index/textpresso.py +0 -0
  31. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/text_preprocessing.py +0 -0
  32. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/text_similarity.py +0 -0
  33. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/scraping.py +0 -0
  34. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/timeout.py +0 -0
  35. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/__init__.py +0 -0
  36. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/abc_entities.py +0 -0
  37. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/abc_search.py +0 -0
  38. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/abc_tag_writer.py +0 -0
  39. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/abc_tags.py +0 -0
  40. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/corpus.py +0 -0
  41. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/person.py +0 -0
  42. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/utils/__init__.py +0 -0
  43. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/utils/alliance.py +0 -0
  44. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/utils/auth_utils.py +0 -0
  45. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools.egg-info/SOURCES.txt +0 -0
  46. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools.egg-info/dependency_links.txt +0 -0
  47. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools.egg-info/requires.txt +0 -0
  48. {wbtools-3.7.0 → wbtools-3.7.2}/wbtools.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: wbtools
3
- Version: 3.7.0
3
+ Version: 3.7.2
4
4
  Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
5
5
  Author-email: Valerio Arnaboldi <valearna@caltech.edu>
6
6
  Project-URL: Homepage, https://github.com/WormBase/wbtools
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "wbtools"
7
- version = "3.7.0"
7
+ version = "3.7.2"
8
8
  authors = [
9
9
  {name = "Valerio Arnaboldi", email = "valearna@caltech.edu"},
10
10
  ]
@@ -36,6 +36,12 @@ logging.getLogger("pdfminer").setLevel(logging.WARNING)
36
36
  ABC_API = os.environ.get('API_SERVER', "literature-rest.alliancegenome.org")
37
37
 
38
38
 
39
+ def get_emails_from_tei_header(tei_content: bytes) -> List[str]:
40
+ header = tei_content.decode("utf-8", "replace").split("</teiHeader>", 1)[0]
41
+ emails = [re.sub('<[^<]+>', '', email).strip() for email in re.findall(r"<email>(.*?)</email>", header, re.S)]
42
+ return list(dict.fromkeys(email for email in emails if email))
43
+
44
+
39
45
  @timeout(3600)
40
46
  def convert_pdf_to_txt(file_path):
41
47
  client = Client(base_url=os.environ.get("GROBID_API_URL", "http://cervino.caltech.edu:8070/api"), timeout=1000,
@@ -49,7 +55,8 @@ def convert_pdf_to_txt(file_path):
49
55
  input_=File(file_name=pdf_file.name, payload=fin, mime_type="application/pdf"))
50
56
  r = process_fulltext_document.sync_detailed(client=client, multipart_data=form)
51
57
  if r.is_success:
52
- article: Article = TEI.parse(r.content, figures=True)
58
+ # back=True keeps the back matter (e.g. data availability, acknowledgements)
59
+ article: Article = TEI.parse(r.content, figures=True, back=True)
53
60
  sentences = []
54
61
  for section in article.sections:
55
62
  # skip sections that have three paragraph with the first and the last being empty. These are
@@ -63,6 +70,8 @@ def convert_pdf_to_txt(file_path):
63
70
  for paragraph in section.paragraphs:
64
71
  for sentence in paragraph:
65
72
  sentences.append(re.sub('<[^<]+>', '', sentence.text))
73
+ # GROBID moves the author emails to the TEI header, which is not part of the sections
74
+ sentences.extend(get_emails_from_tei_header(r.content))
66
75
  return sentences
67
76
  else:
68
77
  return []
@@ -199,6 +208,9 @@ class WBPaper(object):
199
208
  token = get_authentication_token()
200
209
  headers = generate_headers(token)
201
210
  paper_content = get_data_from_url(file_download_api, headers, file_type='pdf')
211
+ if paper_content is None:
212
+ # the download failed (get_data_from_url already logged why): skip this file, keep the others
213
+ return False
202
214
  with tempfile.NamedTemporaryFile() as tmp_file:
203
215
  tmp_file.write(paper_content)
204
216
  text_content = convert_pdf_to_txt(tmp_file.name)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: wbtools
3
- Version: 3.7.0
3
+ Version: 3.7.2
4
4
  Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
5
5
  Author-email: Valerio Arnaboldi <valearna@caltech.edu>
6
6
  Project-URL: Homepage, https://github.com/WormBase/wbtools
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes