wbtools 3.6.0__tar.gz → 3.7.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {wbtools-3.6.0/wbtools.egg-info → wbtools-3.7.1}/PKG-INFO +1 -1
  2. {wbtools-3.6.0 → wbtools-3.7.1}/pyproject.toml +1 -1
  3. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/abc_entities.py +57 -28
  4. wbtools-3.7.1/wbtools/literature/abc_tag_writer.py +61 -0
  5. wbtools-3.7.1/wbtools/literature/abc_tags.py +126 -0
  6. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/paper.py +3 -0
  7. {wbtools-3.6.0 → wbtools-3.7.1/wbtools.egg-info}/PKG-INFO +1 -1
  8. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools.egg-info/SOURCES.txt +2 -0
  9. {wbtools-3.6.0 → wbtools-3.7.1}/LICENSE +0 -0
  10. {wbtools-3.6.0 → wbtools-3.7.1}/README.md +0 -0
  11. {wbtools-3.6.0 → wbtools-3.7.1}/setup.cfg +0 -0
  12. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/__init__.py +0 -0
  13. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/__init__.py +0 -0
  14. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/abstract_manager.py +0 -0
  15. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/afp.py +0 -0
  16. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/antibody.py +0 -0
  17. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/dbmanager.py +0 -0
  18. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/expression.py +0 -0
  19. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/gene.py +0 -0
  20. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/generic.py +0 -0
  21. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/paper.py +0 -0
  22. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/db/person.py +0 -0
  23. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/__init__.py +0 -0
  24. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/email/__init__.py +0 -0
  25. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/email/generic.py +0 -0
  26. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/__init__.py +0 -0
  27. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/common.py +0 -0
  28. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/entity_extraction/__init__.py +0 -0
  29. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/entity_extraction/email_addresses.py +0 -0
  30. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/entity_extraction/ntt_extractor.py +0 -0
  31. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/entity_extraction/variations.py +0 -0
  32. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/literature_index/__init__.py +0 -0
  33. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/literature_index/abstract_index.py +0 -0
  34. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/literature_index/textpresso.py +0 -0
  35. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/text_preprocessing.py +0 -0
  36. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/nlp/text_similarity.py +0 -0
  37. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/scraping.py +0 -0
  38. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/lib/timeout.py +0 -0
  39. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/__init__.py +0 -0
  40. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/abc_search.py +0 -0
  41. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/corpus.py +0 -0
  42. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/literature/person.py +0 -0
  43. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/utils/__init__.py +0 -0
  44. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/utils/alliance.py +0 -0
  45. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools/utils/auth_utils.py +0 -0
  46. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools.egg-info/dependency_links.txt +0 -0
  47. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools.egg-info/requires.txt +0 -0
  48. {wbtools-3.6.0 → wbtools-3.7.1}/wbtools.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: wbtools
3
- Version: 3.6.0
3
+ Version: 3.7.1
4
4
  Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
5
5
  Author-email: Valerio Arnaboldi <valearna@caltech.edu>
6
6
  Project-URL: Homepage, https://github.com/WormBase/wbtools
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "wbtools"
7
- version = "3.6.0"
7
+ version = "3.7.1"
8
8
  authors = [
9
9
  {name = "Valerio Arnaboldi", email = "valearna@caltech.edu"},
10
10
  ]
@@ -1,5 +1,5 @@
1
1
  import logging
2
- from typing import Dict, List, Tuple
2
+ from typing import Dict, List, Optional, Tuple
3
3
 
4
4
  import requests
5
5
 
@@ -25,6 +25,54 @@ ENTITY_PREFIXES = {"gene": "WB:WBGene", "allele": "WB:WBVar", "strain": "WB:WBSt
25
25
  "transgene": "WB:WBTransgene", "species": "NCBITaxon:"}
26
26
 
27
27
 
28
+ def entity_from_tag(tag: dict) -> Optional[Tuple[str, str, Optional[str]]]:
29
+ """get the WB entity named by a topic entity tag, whatever its source method
30
+
31
+ Args:
32
+ tag (dict): a tag returned by /topic_entity_tag/by_references
33
+
34
+ Returns:
35
+ Optional[Tuple[str, str, Optional[str]]]: (entity type, entity curie, entity name), or None for negated tags,
36
+ tags of other MODs, non-entity topics and non-WB entities
37
+ """
38
+ entity_type = TOPIC_ENTITY_TYPES.get(tag.get("topic"))
39
+ entity = tag.get("entity")
40
+ tag_source = tag.get("tag_source") or {}
41
+ if entity_type is None or not entity or tag.get("negated") or \
42
+ tag_source.get("secondary_data_provider_abbreviation", MOD) != MOD or \
43
+ not entity.startswith(ENTITY_PREFIXES[entity_type]):
44
+ return None
45
+ return entity_type, entity, tag.get("entity_name")
46
+
47
+
48
+ def fetch_topic_entity_tags(agr_curies: List[str], filters: dict, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> dict:
49
+ """get the topic entity tags of up to BATCH_SIZE references
50
+
51
+ Args:
52
+ agr_curies (List[str]): AGRKB curies of the references
53
+ filters (dict): the by_references filters (source_methods, topics, mods)
54
+ timeout (int): request timeout in seconds
55
+
56
+ Returns:
57
+ dict: the tags by curie
58
+
59
+ Raises:
60
+ ABCRequestError: if the ABC request fails
61
+ """
62
+ headers = generate_headers(get_authentication_token())
63
+ body = {"curies_or_reference_ids": agr_curies, "filters": filters}
64
+ try:
65
+ response = requests.post(f"https://{ABC_API}/topic_entity_tag/by_references", json=body, headers=headers,
66
+ timeout=timeout)
67
+ response.raise_for_status()
68
+ result = response.json()
69
+ except (requests.exceptions.RequestException, ValueError) as e:
70
+ raise ABCRequestError(f"ABC topic entity tag request failed: {e}") from e
71
+ if not isinstance(result, dict) or not isinstance(result.get("tags"), dict):
72
+ raise ABCRequestError(f"ABC topic entity tag request failed: {result}")
73
+ return result["tags"]
74
+
75
+
28
76
  def get_abc_extracted_entities(agr_curies: List[str]) -> Dict[str, Dict[str, List[Tuple[str, str]]]]:
29
77
  """get the entities extracted by the ABC entity extractor for the given references
30
78
 
@@ -41,37 +89,18 @@ def get_abc_extracted_entities(agr_curies: List[str]) -> Dict[str, Dict[str, Lis
41
89
  entities = {curie: {entity_type: [] for entity_type in ENTITY_TYPES} for curie in agr_curies}
42
90
  for start in range(0, len(agr_curies), BATCH_SIZE):
43
91
  batch = agr_curies[start:start + BATCH_SIZE]
44
- tags_by_curie = _get_entity_tags(batch)
92
+ # scoped to WB: another MOD's extractor can tag the same reference (e.g. with its own species)
93
+ tags_by_curie = fetch_topic_entity_tags(batch, {"source_methods": [ENTITY_EXTRACTOR_SOURCE_METHOD],
94
+ "topics": list(TOPIC_ENTITY_TYPES), "mods": [MOD]})
45
95
  for curie in batch:
46
96
  tags = tags_by_curie.get(curie) or []
47
97
  if not tags:
48
98
  logger.warning(f"No entity extraction tags in ABC for {curie}")
49
99
  for tag in tags:
50
- entity_type = TOPIC_ENTITY_TYPES.get(tag.get("topic"))
51
- entity = tag.get("entity")
52
- tag_source = tag.get("tag_source") or {}
53
- if entity_type is None or not entity or tag.get("negated") or \
54
- tag_source.get("source_method") != ENTITY_EXTRACTOR_SOURCE_METHOD or \
55
- tag_source.get("secondary_data_provider_abbreviation", MOD) != MOD:
100
+ if (tag.get("tag_source") or {}).get("source_method") != ENTITY_EXTRACTOR_SOURCE_METHOD:
56
101
  continue
57
- if entity.startswith(ENTITY_PREFIXES[entity_type]):
58
- entities[curie][entity_type].append((entity, tag.get("entity_name")))
102
+ entity = entity_from_tag(tag)
103
+ if entity:
104
+ entity_type, entity_curie, entity_name = entity
105
+ entities[curie][entity_type].append((entity_curie, entity_name))
59
106
  return entities
60
-
61
-
62
- def _get_entity_tags(agr_curies: List[str]) -> dict:
63
- headers = generate_headers(get_authentication_token())
64
- body = {"curies_or_reference_ids": agr_curies,
65
- # scoped to WB: another MOD's extractor can tag the same reference (e.g. with its own species)
66
- "filters": {"source_methods": [ENTITY_EXTRACTOR_SOURCE_METHOD], "topics": list(TOPIC_ENTITY_TYPES),
67
- "mods": [MOD]}}
68
- try:
69
- response = requests.post(f"https://{ABC_API}/topic_entity_tag/by_references", json=body, headers=headers,
70
- timeout=DEFAULT_REQUEST_TIMEOUT)
71
- response.raise_for_status()
72
- result = response.json()
73
- except (requests.exceptions.RequestException, ValueError) as e:
74
- raise ABCRequestError(f"ABC topic entity tag request failed: {e}") from e
75
- if not isinstance(result, dict) or not isinstance(result.get("tags"), dict):
76
- raise ABCRequestError(f"ABC topic entity tag request failed: {result}")
77
- return result["tags"]
@@ -0,0 +1,61 @@
1
+ import logging
2
+ from typing import List, Optional
3
+
4
+ import requests
5
+
6
+ from wbtools.literature.paper import ABC_API, ABCRequestError, DEFAULT_REQUEST_TIMEOUT
7
+ from wbtools.utils.auth_utils import get_authentication_token, generate_headers
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+ # 409 reasons meaning that the ABC already has the tag
12
+ ALREADY_STORED_REASONS = ("duplicate", "different_creator")
13
+
14
+
15
+ def create_topic_entity_tags(agr_curie: str, tags: List[dict], timeout: int = DEFAULT_REQUEST_TIMEOUT) -> int:
16
+ """create topic entity tags for a reference, with one POST /topic_entity_tag/ per tag
17
+
18
+ Tags that the ABC already has (409 duplicate or different_creator) are skipped, so posting the same tags again
19
+ completes a partial earlier write.
20
+
21
+ Args:
22
+ agr_curie (str): the AGRKB curie of the reference
23
+ tags (List[dict]): the tag payloads, without reference_curie
24
+ timeout (int): request timeout in seconds
25
+
26
+ Returns:
27
+ int: the number of tags created
28
+
29
+ Raises:
30
+ ABCRequestError: if a tag can't be created; the tags posted before it stay in the ABC
31
+ """
32
+ if not tags:
33
+ return 0
34
+ headers = generate_headers(get_authentication_token())
35
+ created = 0
36
+ for tag in tags:
37
+ payload = dict(tag, reference_curie=agr_curie)
38
+ try:
39
+ response = requests.post(f"https://{ABC_API}/topic_entity_tag/", json=payload, headers=headers,
40
+ timeout=timeout)
41
+ except requests.exceptions.RequestException as e:
42
+ raise ABCRequestError(f"ABC tag creation failed for {agr_curie} {_describe(tag)}: {e}") from e
43
+ if response.status_code in (200, 201):
44
+ created += 1
45
+ elif response.status_code != 409 or _conflict_reason(response) not in ALREADY_STORED_REASONS:
46
+ raise ABCRequestError(f"ABC tag creation failed for {agr_curie} {_describe(tag)}: "
47
+ f"HTTP {response.status_code} {response.text[:500]}")
48
+ return created
49
+
50
+
51
+ def _conflict_reason(response) -> Optional[str]:
52
+ try:
53
+ body = response.json()
54
+ except ValueError:
55
+ return None
56
+ detail = body.get("detail") if isinstance(body, dict) else None
57
+ return detail.get("reason") if isinstance(detail, dict) else None
58
+
59
+
60
+ def _describe(tag: dict) -> str:
61
+ return f"(topic {tag.get('topic')}, entity {tag.get('entity')}, negated {tag.get('negated')})"
@@ -0,0 +1,126 @@
1
+ import logging
2
+ from typing import Dict, List, Optional
3
+
4
+ import requests
5
+
6
+ from wbtools.literature.abc_entities import (BATCH_SIZE, ENTITY_EXTRACTOR_SOURCE_METHOD, ENTITY_TYPES, MOD,
7
+ TOPIC_ENTITY_TYPES, entity_from_tag, fetch_topic_entity_tags)
8
+ from wbtools.literature.paper import ABC_API, ABCRequestError, DEFAULT_REQUEST_TIMEOUT
9
+ from wbtools.utils.auth_utils import get_authentication_token, generate_headers
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+ ACK_PIPELINE_SOURCE_METHOD = "ACKnowledge_pipeline"
14
+ DOCUMENT_CLASSIFIER_SOURCE_METHOD = "abc_document_classifier"
15
+ ENTITY_SOURCE_METHODS = (ENTITY_EXTRACTOR_SOURCE_METHOD, ACK_PIPELINE_SOURCE_METHOD)
16
+ CLASSIFICATION_SOURCE_METHODS = (DOCUMENT_CLASSIFIER_SOURCE_METHOD, ACK_PIPELINE_SOURCE_METHOD)
17
+ CLASSIFICATION_TOPICS = (
18
+ "ATP:0000041", # expression
19
+ "ATP:0000061", # catalytic activity
20
+ "ATP:0000068", # genetic interaction
21
+ "ATP:0000069", # physical interaction
22
+ "ATP:0000070", # regulatory interaction
23
+ "ATP:0000082", # RNAi phenotype
24
+ "ATP:0000083", # allele phenotype
25
+ "ATP:0000084", # overexpression phenotype
26
+ )
27
+ # the fields of an entity tag and of a classification tag that are needed to write tags derived from them
28
+ ENTITY_TAG_FIELDS = ("topic", "entity_type", "species", "entity_id_validation", "data_novelty", "data_context")
29
+ CLASSIFICATION_FIELDS = ("confidence_level", "confidence_score", "data_novelty", "data_context", "ml_model_version")
30
+
31
+
32
+ def empty_paper_tags() -> dict:
33
+ """the value get_abc_paper_tags returns for a reference without tags"""
34
+ return {"entities": {source: {entity_type: [] for entity_type in ENTITY_TYPES}
35
+ for source in ENTITY_SOURCE_METHODS},
36
+ "entity_tag_fields": {source: {} for source in ENTITY_SOURCE_METHODS},
37
+ "has_ack_pipeline_entity_tags": False,
38
+ "classifications": {source: {} for source in CLASSIFICATION_SOURCE_METHODS}}
39
+
40
+
41
+ def get_abc_paper_tags(agr_curies: List[str], timeout: int = DEFAULT_REQUEST_TIMEOUT) -> Dict[str, dict]:
42
+ """get the WB entities and classifications of the given references with one ABC call per batch
43
+
44
+ Args:
45
+ agr_curies (List[str]): AGRKB curies of the references
46
+ timeout (int): request timeout in seconds
47
+
48
+ Returns:
49
+ Dict[str, dict]: for each curie:
50
+ "entities": the (entity curie, entity name) pairs by entity type, for each entity source method;
51
+ "entity_tag_fields": the fields of the first tag of each entity, for each entity source method;
52
+ "has_ack_pipeline_entity_tags": whether the reference has any ACKnowledge_pipeline entity tag, negated
53
+ ones included;
54
+ "classifications": the latest tag of each classification topic, for each classification source method
55
+ (highest ml_model_version, then newest; a tag without a version is the oldest).
56
+
57
+ Raises:
58
+ ABCRequestError: if the ABC request fails
59
+ """
60
+ paper_tags = {curie: empty_paper_tags() for curie in agr_curies}
61
+ source_methods = list(dict.fromkeys(ENTITY_SOURCE_METHODS + CLASSIFICATION_SOURCE_METHODS))
62
+ for start in range(0, len(agr_curies), BATCH_SIZE):
63
+ batch = agr_curies[start:start + BATCH_SIZE]
64
+ tags_by_curie = fetch_topic_entity_tags(batch, {
65
+ "source_methods": source_methods, "topics": list(TOPIC_ENTITY_TYPES) + list(CLASSIFICATION_TOPICS),
66
+ "mods": [MOD]}, timeout=timeout)
67
+ for curie in batch:
68
+ for tag in tags_by_curie.get(curie) or []:
69
+ _add_tag(paper_tags[curie], tag)
70
+ return paper_tags
71
+
72
+
73
+ def _add_tag(paper_tags: dict, tag: dict):
74
+ tag_source = tag.get("tag_source") or {}
75
+ source = tag_source.get("source_method")
76
+ topic = tag.get("topic")
77
+ if tag_source.get("secondary_data_provider_abbreviation", MOD) != MOD:
78
+ return
79
+ if topic in TOPIC_ENTITY_TYPES and source in ENTITY_SOURCE_METHODS:
80
+ if source == ACK_PIPELINE_SOURCE_METHOD:
81
+ # negated "no entity found" tags count too: they record that ACKnowledge processed the paper
82
+ paper_tags["has_ack_pipeline_entity_tags"] = True
83
+ entity = entity_from_tag(tag)
84
+ if entity:
85
+ entity_type, entity_curie, entity_name = entity
86
+ paper_tags["entities"][source][entity_type].append((entity_curie, entity_name))
87
+ paper_tags["entity_tag_fields"][source].setdefault(
88
+ entity_curie, {field: tag.get(field) for field in ENTITY_TAG_FIELDS})
89
+ elif topic in CLASSIFICATION_TOPICS and source in CLASSIFICATION_SOURCE_METHODS and not tag.get("entity"):
90
+ classification = {"negated": bool(tag.get("negated")), "date_created": tag.get("date_created") or ""}
91
+ classification.update({field: tag.get(field) for field in CLASSIFICATION_FIELDS})
92
+ classifications = paper_tags["classifications"][source]
93
+ current = classifications.get(topic)
94
+ # a paper can be classified more than once: the latest model wins, then the newest tag
95
+ if current is None or _recency(classification) > _recency(current):
96
+ classifications[topic] = classification
97
+
98
+
99
+ def _recency(classification: dict) -> tuple:
100
+ version = classification["ml_model_version"]
101
+ return (-1 if version is None else version), classification["date_created"]
102
+
103
+
104
+ def get_abc_curie(wb_paper_id: str, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> Optional[str]:
105
+ """get the AGRKB curie of a WB paper
106
+
107
+ Args:
108
+ wb_paper_id (str): the WB paper id without prefix (e.g. "00069459")
109
+ timeout (int): request timeout in seconds
110
+
111
+ Returns:
112
+ Optional[str]: the curie, or None if the paper is not in the ABC
113
+
114
+ Raises:
115
+ ABCRequestError: if the ABC request fails
116
+ """
117
+ headers = generate_headers(get_authentication_token())
118
+ url = f"https://{ABC_API}/reference/by_cross_reference/WB:WBPaper{wb_paper_id}"
119
+ try:
120
+ response = requests.get(url, headers=headers, timeout=timeout)
121
+ if response.status_code == 404:
122
+ return None
123
+ response.raise_for_status()
124
+ return response.json()["curie"]
125
+ except (requests.exceptions.RequestException, ValueError, KeyError, TypeError) as e:
126
+ raise ABCRequestError(f"ABC cross reference request failed for WBPaper{wb_paper_id}: {e}") from e
@@ -199,6 +199,9 @@ class WBPaper(object):
199
199
  token = get_authentication_token()
200
200
  headers = generate_headers(token)
201
201
  paper_content = get_data_from_url(file_download_api, headers, file_type='pdf')
202
+ if paper_content is None:
203
+ # the download failed (get_data_from_url already logged why): skip this file, keep the others
204
+ return False
202
205
  with tempfile.NamedTemporaryFile() as tmp_file:
203
206
  tmp_file.write(paper_content)
204
207
  text_content = convert_pdf_to_txt(tmp_file.name)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: wbtools
3
- Version: 3.6.0
3
+ Version: 3.7.1
4
4
  Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
5
5
  Author-email: Valerio Arnaboldi <valearna@caltech.edu>
6
6
  Project-URL: Homepage, https://github.com/WormBase/wbtools
@@ -36,6 +36,8 @@ wbtools/lib/nlp/literature_index/textpresso.py
36
36
  wbtools/literature/__init__.py
37
37
  wbtools/literature/abc_entities.py
38
38
  wbtools/literature/abc_search.py
39
+ wbtools/literature/abc_tag_writer.py
40
+ wbtools/literature/abc_tags.py
39
41
  wbtools/literature/corpus.py
40
42
  wbtools/literature/paper.py
41
43
  wbtools/literature/person.py
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes