wbtools 3.7.1__tar.gz → 3.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wbtools-3.7.1/wbtools.egg-info → wbtools-3.7.2}/PKG-INFO +1 -1
- {wbtools-3.7.1 → wbtools-3.7.2}/pyproject.toml +1 -1
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/literature/paper.py +10 -1
- {wbtools-3.7.1 → wbtools-3.7.2/wbtools.egg-info}/PKG-INFO +1 -1
- {wbtools-3.7.1 → wbtools-3.7.2}/LICENSE +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/README.md +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/setup.cfg +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/abstract_manager.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/afp.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/antibody.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/dbmanager.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/expression.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/gene.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/generic.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/paper.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/db/person.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/email/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/email/generic.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/common.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/email_addresses.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/ntt_extractor.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/entity_extraction/variations.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/literature_index/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/literature_index/abstract_index.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/literature_index/textpresso.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/text_preprocessing.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/nlp/text_similarity.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/scraping.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/lib/timeout.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/literature/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/literature/abc_entities.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/literature/abc_search.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/literature/abc_tag_writer.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/literature/abc_tags.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/literature/corpus.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/literature/person.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/utils/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/utils/alliance.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools/utils/auth_utils.py +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools.egg-info/SOURCES.txt +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools.egg-info/dependency_links.txt +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools.egg-info/requires.txt +0 -0
- {wbtools-3.7.1 → wbtools-3.7.2}/wbtools.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: wbtools
|
|
3
|
-
Version: 3.7.
|
|
3
|
+
Version: 3.7.2
|
|
4
4
|
Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
|
|
5
5
|
Author-email: Valerio Arnaboldi <valearna@caltech.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/WormBase/wbtools
|
|
@@ -36,6 +36,12 @@ logging.getLogger("pdfminer").setLevel(logging.WARNING)
|
|
|
36
36
|
ABC_API = os.environ.get('API_SERVER', "literature-rest.alliancegenome.org")
|
|
37
37
|
|
|
38
38
|
|
|
39
|
+
def get_emails_from_tei_header(tei_content: bytes) -> List[str]:
|
|
40
|
+
header = tei_content.decode("utf-8", "replace").split("</teiHeader>", 1)[0]
|
|
41
|
+
emails = [re.sub('<[^<]+>', '', email).strip() for email in re.findall(r"<email>(.*?)</email>", header, re.S)]
|
|
42
|
+
return list(dict.fromkeys(email for email in emails if email))
|
|
43
|
+
|
|
44
|
+
|
|
39
45
|
@timeout(3600)
|
|
40
46
|
def convert_pdf_to_txt(file_path):
|
|
41
47
|
client = Client(base_url=os.environ.get("GROBID_API_URL", "http://cervino.caltech.edu:8070/api"), timeout=1000,
|
|
@@ -49,7 +55,8 @@ def convert_pdf_to_txt(file_path):
|
|
|
49
55
|
input_=File(file_name=pdf_file.name, payload=fin, mime_type="application/pdf"))
|
|
50
56
|
r = process_fulltext_document.sync_detailed(client=client, multipart_data=form)
|
|
51
57
|
if r.is_success:
|
|
52
|
-
|
|
58
|
+
# back=True keeps the back matter (e.g. data availability, acknowledgements)
|
|
59
|
+
article: Article = TEI.parse(r.content, figures=True, back=True)
|
|
53
60
|
sentences = []
|
|
54
61
|
for section in article.sections:
|
|
55
62
|
# skip sections that have three paragraph with the first and the last being empty. These are
|
|
@@ -63,6 +70,8 @@ def convert_pdf_to_txt(file_path):
|
|
|
63
70
|
for paragraph in section.paragraphs:
|
|
64
71
|
for sentence in paragraph:
|
|
65
72
|
sentences.append(re.sub('<[^<]+>', '', sentence.text))
|
|
73
|
+
# GROBID moves the author emails to the TEI header, which is not part of the sections
|
|
74
|
+
sentences.extend(get_emails_from_tei_header(r.content))
|
|
66
75
|
return sentences
|
|
67
76
|
else:
|
|
68
77
|
return []
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: wbtools
|
|
3
|
-
Version: 3.7.
|
|
3
|
+
Version: 3.7.2
|
|
4
4
|
Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
|
|
5
5
|
Author-email: Valerio Arnaboldi <valearna@caltech.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/WormBase/wbtools
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|