wbtools 3.7.0__tar.gz → 3.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wbtools-3.7.0/wbtools.egg-info → wbtools-3.7.2}/PKG-INFO +1 -1
- {wbtools-3.7.0 → wbtools-3.7.2}/pyproject.toml +1 -1
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/paper.py +13 -1
- {wbtools-3.7.0 → wbtools-3.7.2/wbtools.egg-info}/PKG-INFO +1 -1
- {wbtools-3.7.0 → wbtools-3.7.2}/LICENSE +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/README.md +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/setup.cfg +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/__init__.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/__init__.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/abstract_manager.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/afp.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/antibody.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/dbmanager.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/expression.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/gene.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/generic.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/paper.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/db/person.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/__init__.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/email/__init__.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/email/generic.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/__init__.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/common.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/__init__.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/email_addresses.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/ntt_extractor.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/variations.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/literature_index/__init__.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/literature_index/abstract_index.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/literature_index/textpresso.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/text_preprocessing.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/nlp/text_similarity.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/scraping.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/lib/timeout.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/__init__.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/abc_entities.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/abc_search.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/abc_tag_writer.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/abc_tags.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/corpus.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/literature/person.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/utils/__init__.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/utils/alliance.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools/utils/auth_utils.py +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools.egg-info/SOURCES.txt +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools.egg-info/dependency_links.txt +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools.egg-info/requires.txt +0 -0
- {wbtools-3.7.0 → wbtools-3.7.2}/wbtools.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: wbtools
|
|
3
|
-
Version: 3.7.
|
|
3
|
+
Version: 3.7.2
|
|
4
4
|
Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
|
|
5
5
|
Author-email: Valerio Arnaboldi <valearna@caltech.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/WormBase/wbtools
|
|
@@ -36,6 +36,12 @@ logging.getLogger("pdfminer").setLevel(logging.WARNING)
|
|
|
36
36
|
ABC_API = os.environ.get('API_SERVER', "literature-rest.alliancegenome.org")
|
|
37
37
|
|
|
38
38
|
|
|
39
|
+
def get_emails_from_tei_header(tei_content: bytes) -> List[str]:
|
|
40
|
+
header = tei_content.decode("utf-8", "replace").split("</teiHeader>", 1)[0]
|
|
41
|
+
emails = [re.sub('<[^<]+>', '', email).strip() for email in re.findall(r"<email>(.*?)</email>", header, re.S)]
|
|
42
|
+
return list(dict.fromkeys(email for email in emails if email))
|
|
43
|
+
|
|
44
|
+
|
|
39
45
|
@timeout(3600)
|
|
40
46
|
def convert_pdf_to_txt(file_path):
|
|
41
47
|
client = Client(base_url=os.environ.get("GROBID_API_URL", "http://cervino.caltech.edu:8070/api"), timeout=1000,
|
|
@@ -49,7 +55,8 @@ def convert_pdf_to_txt(file_path):
|
|
|
49
55
|
input_=File(file_name=pdf_file.name, payload=fin, mime_type="application/pdf"))
|
|
50
56
|
r = process_fulltext_document.sync_detailed(client=client, multipart_data=form)
|
|
51
57
|
if r.is_success:
|
|
52
|
-
|
|
58
|
+
# back=True keeps the back matter (e.g. data availability, acknowledgements)
|
|
59
|
+
article: Article = TEI.parse(r.content, figures=True, back=True)
|
|
53
60
|
sentences = []
|
|
54
61
|
for section in article.sections:
|
|
55
62
|
# skip sections that have three paragraph with the first and the last being empty. These are
|
|
@@ -63,6 +70,8 @@ def convert_pdf_to_txt(file_path):
|
|
|
63
70
|
for paragraph in section.paragraphs:
|
|
64
71
|
for sentence in paragraph:
|
|
65
72
|
sentences.append(re.sub('<[^<]+>', '', sentence.text))
|
|
73
|
+
# GROBID moves the author emails to the TEI header, which is not part of the sections
|
|
74
|
+
sentences.extend(get_emails_from_tei_header(r.content))
|
|
66
75
|
return sentences
|
|
67
76
|
else:
|
|
68
77
|
return []
|
|
@@ -199,6 +208,9 @@ class WBPaper(object):
|
|
|
199
208
|
token = get_authentication_token()
|
|
200
209
|
headers = generate_headers(token)
|
|
201
210
|
paper_content = get_data_from_url(file_download_api, headers, file_type='pdf')
|
|
211
|
+
if paper_content is None:
|
|
212
|
+
# the download failed (get_data_from_url already logged why): skip this file, keep the others
|
|
213
|
+
return False
|
|
202
214
|
with tempfile.NamedTemporaryFile() as tmp_file:
|
|
203
215
|
tmp_file.write(paper_content)
|
|
204
216
|
text_content = convert_pdf_to_txt(tmp_file.name)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: wbtools
|
|
3
|
-
Version: 3.7.
|
|
3
|
+
Version: 3.7.2
|
|
4
4
|
Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
|
|
5
5
|
Author-email: Valerio Arnaboldi <valearna@caltech.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/WormBase/wbtools
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|