wbtools 3.6.0__tar.gz → 3.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wbtools-3.6.0/wbtools.egg-info → wbtools-3.7.1}/PKG-INFO +1 -1
- {wbtools-3.6.0 → wbtools-3.7.1}/pyproject.toml +1 -1
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/abc_entities.py +57 -28
- wbtools-3.7.1/wbtools/literature/abc_tag_writer.py +61 -0
- wbtools-3.7.1/wbtools/literature/abc_tags.py +126 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/paper.py +3 -0
- {wbtools-3.6.0 → wbtools-3.7.1/wbtools.egg-info}/PKG-INFO +1 -1
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools.egg-info/SOURCES.txt +2 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/LICENSE +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/README.md +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/setup.cfg +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/__init__.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/__init__.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/abstract_manager.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/afp.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/antibody.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/dbmanager.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/expression.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/gene.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/generic.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/paper.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/person.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/__init__.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/email/__init__.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/email/generic.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/__init__.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/common.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/entity_extraction/__init__.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/entity_extraction/email_addresses.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/entity_extraction/ntt_extractor.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/entity_extraction/variations.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/literature_index/__init__.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/literature_index/abstract_index.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/literature_index/textpresso.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/text_preprocessing.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/text_similarity.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/scraping.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/timeout.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/__init__.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/abc_search.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/corpus.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/person.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/utils/__init__.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/utils/alliance.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/utils/auth_utils.py +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools.egg-info/dependency_links.txt +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools.egg-info/requires.txt +0 -0
- {wbtools-3.6.0 → wbtools-3.7.1}/wbtools.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: wbtools
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.7.1
|
|
4
4
|
Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
|
|
5
5
|
Author-email: Valerio Arnaboldi <valearna@caltech.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/WormBase/wbtools
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import Dict, List, Tuple
|
|
2
|
+
from typing import Dict, List, Optional, Tuple
|
|
3
3
|
|
|
4
4
|
import requests
|
|
5
5
|
|
|
@@ -25,6 +25,54 @@ ENTITY_PREFIXES = {"gene": "WB:WBGene", "allele": "WB:WBVar", "strain": "WB:WBSt
|
|
|
25
25
|
"transgene": "WB:WBTransgene", "species": "NCBITaxon:"}
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
def entity_from_tag(tag: dict) -> Optional[Tuple[str, str, Optional[str]]]:
|
|
29
|
+
"""get the WB entity named by a topic entity tag, whatever its source method
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
tag (dict): a tag returned by /topic_entity_tag/by_references
|
|
33
|
+
|
|
34
|
+
Returns:
|
|
35
|
+
Optional[Tuple[str, str, Optional[str]]]: (entity type, entity curie, entity name), or None for negated tags,
|
|
36
|
+
tags of other MODs, non-entity topics and non-WB entities
|
|
37
|
+
"""
|
|
38
|
+
entity_type = TOPIC_ENTITY_TYPES.get(tag.get("topic"))
|
|
39
|
+
entity = tag.get("entity")
|
|
40
|
+
tag_source = tag.get("tag_source") or {}
|
|
41
|
+
if entity_type is None or not entity or tag.get("negated") or \
|
|
42
|
+
tag_source.get("secondary_data_provider_abbreviation", MOD) != MOD or \
|
|
43
|
+
not entity.startswith(ENTITY_PREFIXES[entity_type]):
|
|
44
|
+
return None
|
|
45
|
+
return entity_type, entity, tag.get("entity_name")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def fetch_topic_entity_tags(agr_curies: List[str], filters: dict, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> dict:
|
|
49
|
+
"""get the topic entity tags of up to BATCH_SIZE references
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
agr_curies (List[str]): AGRKB curies of the references
|
|
53
|
+
filters (dict): the by_references filters (source_methods, topics, mods)
|
|
54
|
+
timeout (int): request timeout in seconds
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
dict: the tags by curie
|
|
58
|
+
|
|
59
|
+
Raises:
|
|
60
|
+
ABCRequestError: if the ABC request fails
|
|
61
|
+
"""
|
|
62
|
+
headers = generate_headers(get_authentication_token())
|
|
63
|
+
body = {"curies_or_reference_ids": agr_curies, "filters": filters}
|
|
64
|
+
try:
|
|
65
|
+
response = requests.post(f"https://{ABC_API}/topic_entity_tag/by_references", json=body, headers=headers,
|
|
66
|
+
timeout=timeout)
|
|
67
|
+
response.raise_for_status()
|
|
68
|
+
result = response.json()
|
|
69
|
+
except (requests.exceptions.RequestException, ValueError) as e:
|
|
70
|
+
raise ABCRequestError(f"ABC topic entity tag request failed: {e}") from e
|
|
71
|
+
if not isinstance(result, dict) or not isinstance(result.get("tags"), dict):
|
|
72
|
+
raise ABCRequestError(f"ABC topic entity tag request failed: {result}")
|
|
73
|
+
return result["tags"]
|
|
74
|
+
|
|
75
|
+
|
|
28
76
|
def get_abc_extracted_entities(agr_curies: List[str]) -> Dict[str, Dict[str, List[Tuple[str, str]]]]:
|
|
29
77
|
"""get the entities extracted by the ABC entity extractor for the given references
|
|
30
78
|
|
|
@@ -41,37 +89,18 @@ def get_abc_extracted_entities(agr_curies: List[str]) -> Dict[str, Dict[str, Lis
|
|
|
41
89
|
entities = {curie: {entity_type: [] for entity_type in ENTITY_TYPES} for curie in agr_curies}
|
|
42
90
|
for start in range(0, len(agr_curies), BATCH_SIZE):
|
|
43
91
|
batch = agr_curies[start:start + BATCH_SIZE]
|
|
44
|
-
|
|
92
|
+
# scoped to WB: another MOD's extractor can tag the same reference (e.g. with its own species)
|
|
93
|
+
tags_by_curie = fetch_topic_entity_tags(batch, {"source_methods": [ENTITY_EXTRACTOR_SOURCE_METHOD],
|
|
94
|
+
"topics": list(TOPIC_ENTITY_TYPES), "mods": [MOD]})
|
|
45
95
|
for curie in batch:
|
|
46
96
|
tags = tags_by_curie.get(curie) or []
|
|
47
97
|
if not tags:
|
|
48
98
|
logger.warning(f"No entity extraction tags in ABC for {curie}")
|
|
49
99
|
for tag in tags:
|
|
50
|
-
|
|
51
|
-
entity = tag.get("entity")
|
|
52
|
-
tag_source = tag.get("tag_source") or {}
|
|
53
|
-
if entity_type is None or not entity or tag.get("negated") or \
|
|
54
|
-
tag_source.get("source_method") != ENTITY_EXTRACTOR_SOURCE_METHOD or \
|
|
55
|
-
tag_source.get("secondary_data_provider_abbreviation", MOD) != MOD:
|
|
100
|
+
if (tag.get("tag_source") or {}).get("source_method") != ENTITY_EXTRACTOR_SOURCE_METHOD:
|
|
56
101
|
continue
|
|
57
|
-
|
|
58
|
-
|
|
102
|
+
entity = entity_from_tag(tag)
|
|
103
|
+
if entity:
|
|
104
|
+
entity_type, entity_curie, entity_name = entity
|
|
105
|
+
entities[curie][entity_type].append((entity_curie, entity_name))
|
|
59
106
|
return entities
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def _get_entity_tags(agr_curies: List[str]) -> dict:
|
|
63
|
-
headers = generate_headers(get_authentication_token())
|
|
64
|
-
body = {"curies_or_reference_ids": agr_curies,
|
|
65
|
-
# scoped to WB: another MOD's extractor can tag the same reference (e.g. with its own species)
|
|
66
|
-
"filters": {"source_methods": [ENTITY_EXTRACTOR_SOURCE_METHOD], "topics": list(TOPIC_ENTITY_TYPES),
|
|
67
|
-
"mods": [MOD]}}
|
|
68
|
-
try:
|
|
69
|
-
response = requests.post(f"https://{ABC_API}/topic_entity_tag/by_references", json=body, headers=headers,
|
|
70
|
-
timeout=DEFAULT_REQUEST_TIMEOUT)
|
|
71
|
-
response.raise_for_status()
|
|
72
|
-
result = response.json()
|
|
73
|
-
except (requests.exceptions.RequestException, ValueError) as e:
|
|
74
|
-
raise ABCRequestError(f"ABC topic entity tag request failed: {e}") from e
|
|
75
|
-
if not isinstance(result, dict) or not isinstance(result.get("tags"), dict):
|
|
76
|
-
raise ABCRequestError(f"ABC topic entity tag request failed: {result}")
|
|
77
|
-
return result["tags"]
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import List, Optional
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
|
|
6
|
+
from wbtools.literature.paper import ABC_API, ABCRequestError, DEFAULT_REQUEST_TIMEOUT
|
|
7
|
+
from wbtools.utils.auth_utils import get_authentication_token, generate_headers
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
# 409 reasons meaning that the ABC already has the tag
|
|
12
|
+
ALREADY_STORED_REASONS = ("duplicate", "different_creator")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def create_topic_entity_tags(agr_curie: str, tags: List[dict], timeout: int = DEFAULT_REQUEST_TIMEOUT) -> int:
|
|
16
|
+
"""create topic entity tags for a reference, with one POST /topic_entity_tag/ per tag
|
|
17
|
+
|
|
18
|
+
Tags that the ABC already has (409 duplicate or different_creator) are skipped, so posting the same tags again
|
|
19
|
+
completes a partial earlier write.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
agr_curie (str): the AGRKB curie of the reference
|
|
23
|
+
tags (List[dict]): the tag payloads, without reference_curie
|
|
24
|
+
timeout (int): request timeout in seconds
|
|
25
|
+
|
|
26
|
+
Returns:
|
|
27
|
+
int: the number of tags created
|
|
28
|
+
|
|
29
|
+
Raises:
|
|
30
|
+
ABCRequestError: if a tag can't be created; the tags posted before it stay in the ABC
|
|
31
|
+
"""
|
|
32
|
+
if not tags:
|
|
33
|
+
return 0
|
|
34
|
+
headers = generate_headers(get_authentication_token())
|
|
35
|
+
created = 0
|
|
36
|
+
for tag in tags:
|
|
37
|
+
payload = dict(tag, reference_curie=agr_curie)
|
|
38
|
+
try:
|
|
39
|
+
response = requests.post(f"https://{ABC_API}/topic_entity_tag/", json=payload, headers=headers,
|
|
40
|
+
timeout=timeout)
|
|
41
|
+
except requests.exceptions.RequestException as e:
|
|
42
|
+
raise ABCRequestError(f"ABC tag creation failed for {agr_curie} {_describe(tag)}: {e}") from e
|
|
43
|
+
if response.status_code in (200, 201):
|
|
44
|
+
created += 1
|
|
45
|
+
elif response.status_code != 409 or _conflict_reason(response) not in ALREADY_STORED_REASONS:
|
|
46
|
+
raise ABCRequestError(f"ABC tag creation failed for {agr_curie} {_describe(tag)}: "
|
|
47
|
+
f"HTTP {response.status_code} {response.text[:500]}")
|
|
48
|
+
return created
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _conflict_reason(response) -> Optional[str]:
|
|
52
|
+
try:
|
|
53
|
+
body = response.json()
|
|
54
|
+
except ValueError:
|
|
55
|
+
return None
|
|
56
|
+
detail = body.get("detail") if isinstance(body, dict) else None
|
|
57
|
+
return detail.get("reason") if isinstance(detail, dict) else None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _describe(tag: dict) -> str:
|
|
61
|
+
return f"(topic {tag.get('topic')}, entity {tag.get('entity')}, negated {tag.get('negated')})"
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Dict, List, Optional
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
|
|
6
|
+
from wbtools.literature.abc_entities import (BATCH_SIZE, ENTITY_EXTRACTOR_SOURCE_METHOD, ENTITY_TYPES, MOD,
|
|
7
|
+
TOPIC_ENTITY_TYPES, entity_from_tag, fetch_topic_entity_tags)
|
|
8
|
+
from wbtools.literature.paper import ABC_API, ABCRequestError, DEFAULT_REQUEST_TIMEOUT
|
|
9
|
+
from wbtools.utils.auth_utils import get_authentication_token, generate_headers
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
ACK_PIPELINE_SOURCE_METHOD = "ACKnowledge_pipeline"
|
|
14
|
+
DOCUMENT_CLASSIFIER_SOURCE_METHOD = "abc_document_classifier"
|
|
15
|
+
ENTITY_SOURCE_METHODS = (ENTITY_EXTRACTOR_SOURCE_METHOD, ACK_PIPELINE_SOURCE_METHOD)
|
|
16
|
+
CLASSIFICATION_SOURCE_METHODS = (DOCUMENT_CLASSIFIER_SOURCE_METHOD, ACK_PIPELINE_SOURCE_METHOD)
|
|
17
|
+
CLASSIFICATION_TOPICS = (
|
|
18
|
+
"ATP:0000041", # expression
|
|
19
|
+
"ATP:0000061", # catalytic activity
|
|
20
|
+
"ATP:0000068", # genetic interaction
|
|
21
|
+
"ATP:0000069", # physical interaction
|
|
22
|
+
"ATP:0000070", # regulatory interaction
|
|
23
|
+
"ATP:0000082", # RNAi phenotype
|
|
24
|
+
"ATP:0000083", # allele phenotype
|
|
25
|
+
"ATP:0000084", # overexpression phenotype
|
|
26
|
+
)
|
|
27
|
+
# the fields of an entity tag and of a classification tag that are needed to write tags derived from them
|
|
28
|
+
ENTITY_TAG_FIELDS = ("topic", "entity_type", "species", "entity_id_validation", "data_novelty", "data_context")
|
|
29
|
+
CLASSIFICATION_FIELDS = ("confidence_level", "confidence_score", "data_novelty", "data_context", "ml_model_version")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def empty_paper_tags() -> dict:
|
|
33
|
+
"""the value get_abc_paper_tags returns for a reference without tags"""
|
|
34
|
+
return {"entities": {source: {entity_type: [] for entity_type in ENTITY_TYPES}
|
|
35
|
+
for source in ENTITY_SOURCE_METHODS},
|
|
36
|
+
"entity_tag_fields": {source: {} for source in ENTITY_SOURCE_METHODS},
|
|
37
|
+
"has_ack_pipeline_entity_tags": False,
|
|
38
|
+
"classifications": {source: {} for source in CLASSIFICATION_SOURCE_METHODS}}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def get_abc_paper_tags(agr_curies: List[str], timeout: int = DEFAULT_REQUEST_TIMEOUT) -> Dict[str, dict]:
|
|
42
|
+
"""get the WB entities and classifications of the given references with one ABC call per batch
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
agr_curies (List[str]): AGRKB curies of the references
|
|
46
|
+
timeout (int): request timeout in seconds
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
Dict[str, dict]: for each curie:
|
|
50
|
+
"entities": the (entity curie, entity name) pairs by entity type, for each entity source method;
|
|
51
|
+
"entity_tag_fields": the fields of the first tag of each entity, for each entity source method;
|
|
52
|
+
"has_ack_pipeline_entity_tags": whether the reference has any ACKnowledge_pipeline entity tag, negated
|
|
53
|
+
ones included;
|
|
54
|
+
"classifications": the latest tag of each classification topic, for each classification source method
|
|
55
|
+
(highest ml_model_version, then newest; a tag without a version is the oldest).
|
|
56
|
+
|
|
57
|
+
Raises:
|
|
58
|
+
ABCRequestError: if the ABC request fails
|
|
59
|
+
"""
|
|
60
|
+
paper_tags = {curie: empty_paper_tags() for curie in agr_curies}
|
|
61
|
+
source_methods = list(dict.fromkeys(ENTITY_SOURCE_METHODS + CLASSIFICATION_SOURCE_METHODS))
|
|
62
|
+
for start in range(0, len(agr_curies), BATCH_SIZE):
|
|
63
|
+
batch = agr_curies[start:start + BATCH_SIZE]
|
|
64
|
+
tags_by_curie = fetch_topic_entity_tags(batch, {
|
|
65
|
+
"source_methods": source_methods, "topics": list(TOPIC_ENTITY_TYPES) + list(CLASSIFICATION_TOPICS),
|
|
66
|
+
"mods": [MOD]}, timeout=timeout)
|
|
67
|
+
for curie in batch:
|
|
68
|
+
for tag in tags_by_curie.get(curie) or []:
|
|
69
|
+
_add_tag(paper_tags[curie], tag)
|
|
70
|
+
return paper_tags
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _add_tag(paper_tags: dict, tag: dict):
|
|
74
|
+
tag_source = tag.get("tag_source") or {}
|
|
75
|
+
source = tag_source.get("source_method")
|
|
76
|
+
topic = tag.get("topic")
|
|
77
|
+
if tag_source.get("secondary_data_provider_abbreviation", MOD) != MOD:
|
|
78
|
+
return
|
|
79
|
+
if topic in TOPIC_ENTITY_TYPES and source in ENTITY_SOURCE_METHODS:
|
|
80
|
+
if source == ACK_PIPELINE_SOURCE_METHOD:
|
|
81
|
+
# negated "no entity found" tags count too: they record that ACKnowledge processed the paper
|
|
82
|
+
paper_tags["has_ack_pipeline_entity_tags"] = True
|
|
83
|
+
entity = entity_from_tag(tag)
|
|
84
|
+
if entity:
|
|
85
|
+
entity_type, entity_curie, entity_name = entity
|
|
86
|
+
paper_tags["entities"][source][entity_type].append((entity_curie, entity_name))
|
|
87
|
+
paper_tags["entity_tag_fields"][source].setdefault(
|
|
88
|
+
entity_curie, {field: tag.get(field) for field in ENTITY_TAG_FIELDS})
|
|
89
|
+
elif topic in CLASSIFICATION_TOPICS and source in CLASSIFICATION_SOURCE_METHODS and not tag.get("entity"):
|
|
90
|
+
classification = {"negated": bool(tag.get("negated")), "date_created": tag.get("date_created") or ""}
|
|
91
|
+
classification.update({field: tag.get(field) for field in CLASSIFICATION_FIELDS})
|
|
92
|
+
classifications = paper_tags["classifications"][source]
|
|
93
|
+
current = classifications.get(topic)
|
|
94
|
+
# a paper can be classified more than once: the latest model wins, then the newest tag
|
|
95
|
+
if current is None or _recency(classification) > _recency(current):
|
|
96
|
+
classifications[topic] = classification
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _recency(classification: dict) -> tuple:
|
|
100
|
+
version = classification["ml_model_version"]
|
|
101
|
+
return (-1 if version is None else version), classification["date_created"]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def get_abc_curie(wb_paper_id: str, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> Optional[str]:
|
|
105
|
+
"""get the AGRKB curie of a WB paper
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
wb_paper_id (str): the WB paper id without prefix (e.g. "00069459")
|
|
109
|
+
timeout (int): request timeout in seconds
|
|
110
|
+
|
|
111
|
+
Returns:
|
|
112
|
+
Optional[str]: the curie, or None if the paper is not in the ABC
|
|
113
|
+
|
|
114
|
+
Raises:
|
|
115
|
+
ABCRequestError: if the ABC request fails
|
|
116
|
+
"""
|
|
117
|
+
headers = generate_headers(get_authentication_token())
|
|
118
|
+
url = f"https://{ABC_API}/reference/by_cross_reference/WB:WBPaper{wb_paper_id}"
|
|
119
|
+
try:
|
|
120
|
+
response = requests.get(url, headers=headers, timeout=timeout)
|
|
121
|
+
if response.status_code == 404:
|
|
122
|
+
return None
|
|
123
|
+
response.raise_for_status()
|
|
124
|
+
return response.json()["curie"]
|
|
125
|
+
except (requests.exceptions.RequestException, ValueError, KeyError, TypeError) as e:
|
|
126
|
+
raise ABCRequestError(f"ABC cross reference request failed for WBPaper{wb_paper_id}: {e}") from e
|
|
@@ -199,6 +199,9 @@ class WBPaper(object):
|
|
|
199
199
|
token = get_authentication_token()
|
|
200
200
|
headers = generate_headers(token)
|
|
201
201
|
paper_content = get_data_from_url(file_download_api, headers, file_type='pdf')
|
|
202
|
+
if paper_content is None:
|
|
203
|
+
# the download failed (get_data_from_url already logged why): skip this file, keep the others
|
|
204
|
+
return False
|
|
202
205
|
with tempfile.NamedTemporaryFile() as tmp_file:
|
|
203
206
|
tmp_file.write(paper_content)
|
|
204
207
|
text_content = convert_pdf_to_txt(tmp_file.name)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: wbtools
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.7.1
|
|
4
4
|
Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
|
|
5
5
|
Author-email: Valerio Arnaboldi <valearna@caltech.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/WormBase/wbtools
|
|
@@ -36,6 +36,8 @@ wbtools/lib/nlp/literature_index/textpresso.py
|
|
|
36
36
|
wbtools/literature/__init__.py
|
|
37
37
|
wbtools/literature/abc_entities.py
|
|
38
38
|
wbtools/literature/abc_search.py
|
|
39
|
+
wbtools/literature/abc_tag_writer.py
|
|
40
|
+
wbtools/literature/abc_tags.py
|
|
39
41
|
wbtools/literature/corpus.py
|
|
40
42
|
wbtools/literature/paper.py
|
|
41
43
|
wbtools/literature/person.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|