wbtools 3.7.1__tar.gz → 3.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wbtools-3.7.1/wbtools.egg-info → wbtools-3.8.0}/PKG-INFO +3 -2
- {wbtools-3.7.1 → wbtools-3.8.0}/pyproject.toml +1 -1
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/dbmanager.py +16 -0
- wbtools-3.8.0/wbtools/literature/ack_form/__init__.py +1 -0
- wbtools-3.8.0/wbtools/literature/ack_form/abc.py +82 -0
- wbtools-3.8.0/wbtools/literature/ack_form/author.py +183 -0
- wbtools-3.8.0/wbtools/literature/ack_form/caltech.py +220 -0
- wbtools-3.8.0/wbtools/literature/ack_form/changes.py +147 -0
- wbtools-3.8.0/wbtools/literature/ack_form/constants.py +108 -0
- wbtools-3.8.0/wbtools/literature/ack_form/entities.py +92 -0
- wbtools-3.8.0/wbtools/literature/ack_form/paper_data.py +80 -0
- wbtools-3.8.0/wbtools/literature/ack_form/reverse.py +134 -0
- wbtools-3.8.0/wbtools/literature/ack_form/tags.py +46 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/literature/paper.py +10 -1
- {wbtools-3.7.1 → wbtools-3.8.0/wbtools.egg-info}/PKG-INFO +3 -2
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools.egg-info/SOURCES.txt +10 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/LICENSE +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/README.md +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/setup.cfg +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/abstract_manager.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/afp.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/antibody.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/expression.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/gene.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/generic.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/paper.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/db/person.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/email/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/email/generic.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/common.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/entity_extraction/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/entity_extraction/email_addresses.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/entity_extraction/ntt_extractor.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/entity_extraction/variations.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/literature_index/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/literature_index/abstract_index.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/literature_index/textpresso.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/text_preprocessing.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/nlp/text_similarity.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/scraping.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/lib/timeout.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/literature/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/literature/abc_entities.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/literature/abc_search.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/literature/abc_tag_writer.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/literature/abc_tags.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/literature/corpus.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/literature/person.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/utils/__init__.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/utils/alliance.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools/utils/auth_utils.py +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools.egg-info/dependency_links.txt +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools.egg-info/requires.txt +0 -0
- {wbtools-3.7.1 → wbtools-3.8.0}/wbtools.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: wbtools
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.8.0
|
|
4
4
|
Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
|
|
5
5
|
Author-email: Valerio Arnaboldi <valearna@caltech.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/WormBase/wbtools
|
|
@@ -26,6 +26,7 @@ Requires-Dist: python-dotenv~=1.0.0
|
|
|
26
26
|
Requires-Dist: grobid-client
|
|
27
27
|
Requires-Dist: agr-cognito-py
|
|
28
28
|
Requires-Dist: agr-abc-document-parsers>=1.7.2
|
|
29
|
+
Dynamic: license-file
|
|
29
30
|
|
|
30
31
|
# WBtools
|
|
31
32
|
> Interface to WormBase curation database and Text Mining functions
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
from contextlib import contextmanager
|
|
2
3
|
|
|
3
4
|
import psycopg2
|
|
4
5
|
|
|
@@ -46,3 +47,18 @@ class WBDBManager(AbstractWBDBManager):
|
|
|
46
47
|
|
|
47
48
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
48
49
|
super(WBDBManager, self).__exit__(exc_type, exc_val, exc_tb)
|
|
50
|
+
|
|
51
|
+
@contextmanager
|
|
52
|
+
def transaction(self):
|
|
53
|
+
"""one transaction for every sub-manager: committed on success, rolled back on any error (unlike
|
|
54
|
+
``with manager:``, which commits even when the block raised), and the connection always closed"""
|
|
55
|
+
self.__enter__()
|
|
56
|
+
try:
|
|
57
|
+
yield self
|
|
58
|
+
self.conn.commit()
|
|
59
|
+
except BaseException:
|
|
60
|
+
self.conn.rollback()
|
|
61
|
+
raise
|
|
62
|
+
finally:
|
|
63
|
+
self.curs.close()
|
|
64
|
+
self.conn.close()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""ACKnowledge_form tags: the author answers of the ACKnowledge form as ABC topic entity tags (SCRUM-4141)."""
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""The ABC calls of an ACKnowledge save: read a paper's ACKnowledge_form tags, send the save's changes as one batch
|
|
2
|
+
(POST /topic_entity_tag/batch, SCRUM-4141)."""
|
|
3
|
+
from typing import Dict, List
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
from wbtools.literature.ack_form.changes import TagBatch
|
|
8
|
+
from wbtools.literature.ack_form.constants import ACK_FORM, SOURCES
|
|
9
|
+
from wbtools.literature.paper import ABC_API, ABCRequestError
|
|
10
|
+
from wbtools.utils.auth_utils import generate_headers, get_authentication_token
|
|
11
|
+
|
|
12
|
+
# the ABC refuses larger batches; refusing here saves the call
|
|
13
|
+
MAX_BATCH_CHANGES = 5000
|
|
14
|
+
_SOURCE_IDS: Dict[str, int] = {}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _post(path: str, body: dict, timeout: int) -> requests.Response:
|
|
18
|
+
headers = generate_headers(get_authentication_token())
|
|
19
|
+
try:
|
|
20
|
+
return requests.post(f"https://{ABC_API}{path}", json=body, headers=headers, timeout=timeout)
|
|
21
|
+
except requests.exceptions.RequestException as e:
|
|
22
|
+
raise ABCRequestError(f"ABC {path} failed: {e}") from e
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_ack_form_source_id(timeout: int) -> int:
|
|
26
|
+
"""the id of the WB ACKnowledge_form tag source, looked up once per process
|
|
27
|
+
|
|
28
|
+
Raises:
|
|
29
|
+
ABCRequestError: if the ABC can't be read
|
|
30
|
+
"""
|
|
31
|
+
if ACK_FORM not in _SOURCE_IDS:
|
|
32
|
+
path = f"/tag_source/{SOURCES[ACK_FORM]}/{ACK_FORM}/WB/WB"
|
|
33
|
+
try:
|
|
34
|
+
response = requests.get(f"https://{ABC_API}{path}",
|
|
35
|
+
headers=generate_headers(get_authentication_token()), timeout=timeout)
|
|
36
|
+
except requests.exceptions.RequestException as e:
|
|
37
|
+
raise ABCRequestError(f"ABC {path} failed: {e}") from e
|
|
38
|
+
if response.status_code != 200:
|
|
39
|
+
raise ABCRequestError(f"ABC {path} failed: HTTP {response.status_code} {response.text[:200]}")
|
|
40
|
+
_SOURCE_IDS[ACK_FORM] = response.json()["tag_source_id"]
|
|
41
|
+
return _SOURCE_IDS[ACK_FORM]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def get_ack_form_tags(agr_curie: str, timeout: int) -> List[dict]:
|
|
45
|
+
"""the paper's ACKnowledge_form tags, every contributor's copy
|
|
46
|
+
|
|
47
|
+
Raises:
|
|
48
|
+
ABCRequestError: if the ABC can't be read
|
|
49
|
+
"""
|
|
50
|
+
response = _post("/topic_entity_tag/by_references", {
|
|
51
|
+
"curies_or_reference_ids": [agr_curie], "filters": {"source_methods": [ACK_FORM]}}, timeout)
|
|
52
|
+
if response.status_code != 200:
|
|
53
|
+
raise ABCRequestError(f"ABC by_references failed for {agr_curie}: HTTP {response.status_code} "
|
|
54
|
+
f"{response.text[:500]}")
|
|
55
|
+
try:
|
|
56
|
+
tags = response.json()["tags"]
|
|
57
|
+
except (ValueError, KeyError, TypeError) as e:
|
|
58
|
+
raise ABCRequestError(f"ABC by_references gave an unexpected answer for {agr_curie}: {e}") from e
|
|
59
|
+
return tags.get(agr_curie) or []
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def send_tag_batch(agr_curie: str, batch: TagBatch, timeout: int) -> List[int]:
|
|
63
|
+
"""apply a save's tag changes in one ABC transaction; returns the ids of the created tags
|
|
64
|
+
|
|
65
|
+
Raises:
|
|
66
|
+
ABCRequestError: if the batch is over the cap or the ABC refuses it or can't be reached (the ABC then stored
|
|
67
|
+
nothing, unless the answer was lost after the ABC committed)
|
|
68
|
+
"""
|
|
69
|
+
if batch.size() > MAX_BATCH_CHANGES:
|
|
70
|
+
raise ABCRequestError(f"{batch.size()} tag changes for {agr_curie}; the ABC accepts at most "
|
|
71
|
+
f"{MAX_BATCH_CHANGES} in one batch")
|
|
72
|
+
body = {"reference_curie": agr_curie,
|
|
73
|
+
"create": [dict(tag, reference_curie=agr_curie) for tag in batch.create],
|
|
74
|
+
"update": batch.update, "delete": batch.delete}
|
|
75
|
+
response = _post("/topic_entity_tag/batch", body, timeout)
|
|
76
|
+
if response.status_code != 200:
|
|
77
|
+
raise ABCRequestError(f"ABC tag batch refused for {agr_curie}: HTTP {response.status_code} "
|
|
78
|
+
f"{response.text[:1000]}")
|
|
79
|
+
try:
|
|
80
|
+
return response.json()["created"]
|
|
81
|
+
except (ValueError, KeyError, TypeError) as e:
|
|
82
|
+
raise ABCRequestError(f"ABC tag batch gave an unexpected answer for {agr_curie}: {e}") from e
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""The ACKnowledge_form tags of a paper's ACKnowledge submission, final or partial (spec amendment 12)."""
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
from typing import List, Optional, Tuple
|
|
5
|
+
|
|
6
|
+
from wbtools.literature.ack_form.caltech import CaltechData
|
|
7
|
+
from wbtools.literature.ack_form.constants import (ACK_FORM, ACK_START, DATATYPE_FLAGS, EMPTY_OTHER,
|
|
8
|
+
ENTITY_TYPES, NOVELTY_EXISTING, NOVELTY_NEW_TO_DB)
|
|
9
|
+
from wbtools.literature.ack_form.entities import other_names, other_note, parse_entities
|
|
10
|
+
from wbtools.literature.ack_form.tags import author_key, make_tag
|
|
11
|
+
|
|
12
|
+
AUTHOR_TABLES = [table for entity_type in ENTITY_TYPES.values()
|
|
13
|
+
for table in (entity_type.afp_table, entity_type.other_table) if table] + \
|
|
14
|
+
[flag.table for flag in DATATYPE_FLAGS]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _latest_submission(joinkey: str, data: CaltechData) -> str:
|
|
18
|
+
timestamps = [data.tables.get(table, {})[joinkey].timestamp for table in AUTHOR_TABLES
|
|
19
|
+
if joinkey in data.tables.get(table, {})]
|
|
20
|
+
timestamps.append(data.lasttouched.get(joinkey, ""))
|
|
21
|
+
return max(timestamps)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def in_author_scope(joinkey: str, data: CaltechData) -> bool:
|
|
25
|
+
"""an ACKnowledge submission, final or partial (old author first pass submissions are out of scope)"""
|
|
26
|
+
return joinkey in data.ack_papers and _latest_submission(joinkey, data)[:10] >= ACK_START
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def authors(joinkey: str, data: CaltechData) -> List[Tuple[str, Optional[str]]]:
|
|
30
|
+
"""(WBPerson, contribution timestamp), sorted; unknown_author when the paper has no contributor"""
|
|
31
|
+
contributors = data.contributors.get(joinkey) or {}
|
|
32
|
+
return sorted(contributors.items()) or [("unknown_author", None)]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _date(contributed: Optional[str], row_ts: Optional[str], joinkey: str, data: CaltechData) -> Optional[str]:
|
|
36
|
+
return contributed or row_ts or data.lasttouched.get(joinkey) or None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _note(entities) -> str:
|
|
40
|
+
return "\n".join(f"{e.published_as} ( {e.curie} )" if e.published_as else e.curie for e in entities)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def author_entity_tags(joinkey: str, curie: str, data: CaltechData) -> Tuple[List[dict], List[Tuple[str, str]]]:
|
|
44
|
+
"""the expected ACKnowledge_form entity tags of a paper, and the problems found as (topic, message)"""
|
|
45
|
+
if not in_author_scope(joinkey, data):
|
|
46
|
+
return [], []
|
|
47
|
+
tags: List[dict] = []
|
|
48
|
+
problems: List[Tuple[str, str]] = []
|
|
49
|
+
paper_authors = authors(joinkey, data)
|
|
50
|
+
for entity_type in ENTITY_TYPES.values():
|
|
51
|
+
afp = data.tables.get(entity_type.afp_table, {}).get(joinkey)
|
|
52
|
+
other = data.tables.get(entity_type.other_table, {}).get(joinkey) if entity_type.other_table else None
|
|
53
|
+
tfp = data.tables.get(entity_type.tfp_table, {}).get(joinkey)
|
|
54
|
+
afp_entities, removed = [], []
|
|
55
|
+
if afp is not None and afp.data:
|
|
56
|
+
afp_entities, afp_problems = parse_entities(entity_type.name, afp.data, data)
|
|
57
|
+
problems.extend((entity_type.topic, f"{entity_type.afp_table}: {p}") for p in afp_problems)
|
|
58
|
+
# an unresolved author item could be any of the pipeline entities: no "removed" negations then
|
|
59
|
+
if tfp is not None and tfp.data and not afp_problems:
|
|
60
|
+
kept = {e.curie for e in afp_entities}
|
|
61
|
+
removed = [e for e in parse_entities(entity_type.name, tfp.data, data)[0] if e.curie not in kept]
|
|
62
|
+
names = other_names(other.data) if other is not None else []
|
|
63
|
+
for who, contributed in paper_authors:
|
|
64
|
+
def tag(row_ts, **fields):
|
|
65
|
+
tags.append(make_tag(reference_curie=curie, source_method=ACK_FORM, topic=entity_type.topic,
|
|
66
|
+
created_by=who, date_created=_date(contributed, row_ts, joinkey, data),
|
|
67
|
+
**fields))
|
|
68
|
+
|
|
69
|
+
for entity in afp_entities:
|
|
70
|
+
tag(afp.timestamp, entity_type=entity_type.topic, entity=entity.curie, species=entity.species,
|
|
71
|
+
entity_id_validation="alliance", negated=False, data_novelty=NOVELTY_EXISTING,
|
|
72
|
+
note=_note(afp_entities))
|
|
73
|
+
for entity in removed:
|
|
74
|
+
tag(afp.timestamp, entity_type=entity_type.topic, entity=entity.curie, species=entity.species,
|
|
75
|
+
entity_id_validation="alliance", negated=True, data_novelty=NOVELTY_EXISTING)
|
|
76
|
+
if afp is not None and afp.data == "":
|
|
77
|
+
tag(afp.timestamp, negated=True, data_novelty=NOVELTY_EXISTING)
|
|
78
|
+
if names:
|
|
79
|
+
tag(other.timestamp, negated=False, data_novelty=NOVELTY_NEW_TO_DB,
|
|
80
|
+
note=other_note(other.data) or "\n".join(names))
|
|
81
|
+
elif other is not None and other.data == EMPTY_OTHER:
|
|
82
|
+
tag(other.timestamp, negated=True, data_novelty=NOVELTY_NEW_TO_DB)
|
|
83
|
+
return tags, problems
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _otherantibody(value: str) -> Tuple[bool, str]:
|
|
87
|
+
try:
|
|
88
|
+
entries = json.loads(value or "")
|
|
89
|
+
except ValueError:
|
|
90
|
+
return True, ""
|
|
91
|
+
if not isinstance(entries, list):
|
|
92
|
+
return True, ""
|
|
93
|
+
parts = []
|
|
94
|
+
for entry in entries:
|
|
95
|
+
if not isinstance(entry, dict):
|
|
96
|
+
continue
|
|
97
|
+
fields = [str(entry[key]) for key in ("name", "publicationId") if entry.get(key) not in (None, "")]
|
|
98
|
+
if fields:
|
|
99
|
+
parts.append("; ".join(fields))
|
|
100
|
+
note = " | ".join(parts)
|
|
101
|
+
return note == "", note
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _humandisease(value: Optional[str]) -> Tuple[bool, str]:
|
|
105
|
+
if not value:
|
|
106
|
+
return True, ""
|
|
107
|
+
try:
|
|
108
|
+
parsed = json.loads(value)
|
|
109
|
+
except ValueError:
|
|
110
|
+
return False, _free_text(value)
|
|
111
|
+
if not isinstance(parsed, dict):
|
|
112
|
+
return False, _free_text(value)
|
|
113
|
+
# SCRUM-4141: the first line is the comment (empty when there is none), each further line a disease, so the
|
|
114
|
+
# answer can be rebuilt from the note (reverse.py). The author's own newlines become spaces: a newline in the
|
|
115
|
+
# note is only ever a separator.
|
|
116
|
+
comment = parsed.get("comment")
|
|
117
|
+
comment = "" if comment in (None, "", "checked") else _one_line(str(comment))
|
|
118
|
+
diseases = []
|
|
119
|
+
for disease in parsed.get("diseases") or []:
|
|
120
|
+
disease = re.sub(r"\s+\)", ")", re.sub(r"\(\s+", "(", _one_line(str(disease))))
|
|
121
|
+
if disease:
|
|
122
|
+
diseases.append(disease)
|
|
123
|
+
return not parsed.get("checked"), "\n".join([comment] + diseases) if diseases else comment
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _free_text(value: str) -> str:
|
|
127
|
+
"""a disease answer that is not the form's JSON, on one line as every other flag note"""
|
|
128
|
+
note = value.replace("\n", " ")
|
|
129
|
+
return note[:-1] if note.endswith(" ") else note
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _one_line(text: str) -> str:
|
|
133
|
+
return text.replace("\r", " ").replace("\n", " ").rstrip()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def convert_flag(datatype: str, value: Optional[str]) -> Tuple[bool, str]:
|
|
137
|
+
"""(negated, note) of an afp_<datatype> value, as convertAckToNote in populate_classifier_topic_entity.pl"""
|
|
138
|
+
if datatype == "otherantibody":
|
|
139
|
+
negated, note = _otherantibody(value or "")
|
|
140
|
+
elif datatype == "humandisease":
|
|
141
|
+
return _humandisease(value) # its newlines are separators; its text is already on one line
|
|
142
|
+
else:
|
|
143
|
+
negated = value in ("", None, "0")
|
|
144
|
+
note = "" if negated else value
|
|
145
|
+
note = note.replace("\n", " ")
|
|
146
|
+
return negated, note[:-1] if note.endswith(" ") else note
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def author_flag_tags(joinkey: str, curie: str, data: CaltechData) -> List[dict]:
|
|
150
|
+
"""the expected ACKnowledge_form data-type flag tags of a paper: rows dated from ACK_START on, as
|
|
151
|
+
populate_classifier_topic_entity.pl decides (early ACKnowledge submissions have no entity sections, so the
|
|
152
|
+
ACKnowledge section test of the entity tags does not apply here)"""
|
|
153
|
+
tags = []
|
|
154
|
+
for flag in DATATYPE_FLAGS:
|
|
155
|
+
row = data.tables.get(flag.table, {}).get(joinkey)
|
|
156
|
+
if row is None or row.timestamp[:10] < ACK_START:
|
|
157
|
+
continue
|
|
158
|
+
negated, note = convert_flag(flag.datatype, row.data)
|
|
159
|
+
for who, contributed in authors(joinkey, data):
|
|
160
|
+
tags.append(make_tag(reference_curie=curie, source_method=ACK_FORM, topic=flag.topic, negated=negated,
|
|
161
|
+
data_novelty=flag.novelty, note=note or None, created_by=who,
|
|
162
|
+
date_created=_date(contributed, row.timestamp, joinkey, data)))
|
|
163
|
+
return _merge_notes(tags)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _merge_notes(tags: List[dict]) -> List[dict]:
|
|
167
|
+
"""two flags of one topic (marker and otherexpr) give tags that differ only in their note; the ABC stores those
|
|
168
|
+
as one tag, the first one created, with the notes joined by " | " (create_tag appends the note), so the
|
|
169
|
+
expected tags are merged the same way"""
|
|
170
|
+
merged: dict = {}
|
|
171
|
+
for tag in tags:
|
|
172
|
+
first = merged.setdefault(author_key({**tag, "note": None}), tag)
|
|
173
|
+
if first is not tag and tag.get("note"):
|
|
174
|
+
notes = first["note"].split(" | ") if first.get("note") else []
|
|
175
|
+
if tag["note"] not in notes:
|
|
176
|
+
first["note"] = " | ".join(notes + [tag["note"]])
|
|
177
|
+
return list(merged.values())
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def author_tags(joinkey: str, curie: str, data: CaltechData) -> Tuple[List[dict], List[Tuple[str, str]]]:
|
|
181
|
+
"""all the expected ACKnowledge_form tags of a paper (entities, then data-type flags), and the problems found"""
|
|
182
|
+
entity_tags, problems = author_entity_tags(joinkey, curie, data)
|
|
183
|
+
return entity_tags + author_flag_tags(joinkey, curie, data), problems
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Read the Caltech tables the sync needs into a CaltechData (read only)."""
|
|
2
|
+
import re
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from datetime import datetime, timezone
|
|
5
|
+
from typing import Dict, List, NamedTuple, Optional, Set
|
|
6
|
+
|
|
7
|
+
from wbtools.literature.ack_form.constants import (ACK_SECTION_TABLES, AUTHOR_EVIDENCE, DATATYPE_FLAGS,
|
|
8
|
+
ENTITY_TYPES, PAPER_EDITOR_TABLES)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Row(NamedTuple):
|
|
12
|
+
data: str
|
|
13
|
+
timestamp: str
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Removal(NamedTuple):
|
|
17
|
+
"""an author's paper editor entry (pap_gene/pap_species "from author first pass") a curator removed"""
|
|
18
|
+
value: str # the pap_* value: an 8-digit gene number or an NCBI taxon id
|
|
19
|
+
curator: str # WBPerson
|
|
20
|
+
timestamp: str
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class CaltechData:
|
|
25
|
+
valid: Set[str] = field(default_factory=set)
|
|
26
|
+
merged_into: Dict[str, str] = field(default_factory=dict) # merged paper -> paper it was merged into
|
|
27
|
+
agrkb: Dict[str, str] = field(default_factory=dict)
|
|
28
|
+
lasttouched: Dict[str, str] = field(default_factory=dict) # final ACKnowledge submissions
|
|
29
|
+
ack_papers: Set[str] = field(default_factory=set)
|
|
30
|
+
contributors: Dict[str, Dict[str, str]] = field(default_factory=dict) # paper -> WBPerson -> timestamp
|
|
31
|
+
tables: Dict[str, Dict[str, Row]] = field(default_factory=dict) # tfp_*/afp_* table -> paper -> row
|
|
32
|
+
blackbox: Dict[str, Dict[str, str]] = field(default_factory=dict) # paper -> datatype -> level
|
|
33
|
+
gene_taxon: Dict[str, str] = field(default_factory=dict)
|
|
34
|
+
variation_taxon: Dict[str, str] = field(default_factory=dict)
|
|
35
|
+
strain_by_name: Dict[str, str] = field(default_factory=dict)
|
|
36
|
+
strain_taxon: Dict[str, str] = field(default_factory=dict)
|
|
37
|
+
transgene_by_name: Dict[str, str] = field(default_factory=dict)
|
|
38
|
+
transgene_taxon: Dict[str, str] = field(default_factory=dict)
|
|
39
|
+
species_taxon: Dict[str, str] = field(default_factory=dict) # species name -> NCBITaxon curie
|
|
40
|
+
removals: Dict[str, Dict[str, List[Removal]]] = field(default_factory=dict) # gene/species -> paper -> removals
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
ENTITY_TABLES = [table for entity_type in ENTITY_TYPES.values()
|
|
44
|
+
for table in (entity_type.tfp_table, entity_type.afp_table, entity_type.other_table) if table]
|
|
45
|
+
FLAG_TABLES = [flag.table for flag in DATATYPE_FLAGS]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _table_query(table: str) -> str:
|
|
49
|
+
prefix = table.split("_")[0]
|
|
50
|
+
return f"SELECT joinkey, {table}, {prefix}_timestamp FROM {table} ORDER BY {prefix}_timestamp" # noqa: S608
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
QUERIES: Dict[str, str] = {
|
|
54
|
+
"valid": "SELECT joinkey FROM pap_status WHERE pap_status = 'valid'",
|
|
55
|
+
"merges": "SELECT joinkey, pap_identifier FROM pap_identifier WHERE pap_identifier ~ '^[0-9]{8}$'",
|
|
56
|
+
"agrkb": "SELECT joinkey, pap_identifier FROM pap_identifier WHERE pap_identifier ~ 'AGRKB'",
|
|
57
|
+
"lasttouched": "SELECT joinkey, afp_timestamp FROM afp_lasttouched ORDER BY afp_timestamp",
|
|
58
|
+
# the people who contributed to the submission (not pap_curator, which names only the one who submitted)
|
|
59
|
+
"contributors_afp": "SELECT joinkey, afp_contributor, afp_timestamp FROM afp_contributor ORDER BY afp_timestamp",
|
|
60
|
+
"blackbox": "SELECT cur_paper, cur_datatype, cur_blackbox, cur_timestamp FROM cur_blackbox ORDER BY cur_timestamp",
|
|
61
|
+
"species_index": "SELECT joinkey, pap_species_index, pap_timestamp FROM pap_species_index ORDER BY pap_timestamp",
|
|
62
|
+
"genes": "SELECT joinkey, gin_species FROM gin_species",
|
|
63
|
+
"variations": "SELECT joinkey, obo_data_variation FROM obo_data_variation",
|
|
64
|
+
"strains": "SELECT joinkey, obo_data_strain FROM obo_data_strain",
|
|
65
|
+
"transgenes": "SELECT n.trp_name, p.trp_publicname, s.trp_species FROM trp_name n "
|
|
66
|
+
"LEFT JOIN trp_publicname p USING (joinkey) LEFT JOIN trp_species s USING (joinkey)",
|
|
67
|
+
}
|
|
68
|
+
QUERIES.update({table: _table_query(table) for table in ENTITY_TABLES + FLAG_TABLES})
|
|
69
|
+
for _name, _table in PAPER_EDITOR_TABLES.items():
|
|
70
|
+
QUERIES[f"history_{_name}"] = (f"SELECT joinkey, {_table}, pap_order, pap_curator, pap_evidence, pap_timestamp "
|
|
71
|
+
f"FROM h_{_table} ORDER BY pap_timestamp") # noqa: S608
|
|
72
|
+
QUERIES[f"current_{_name}"] = f"SELECT joinkey, {_table} FROM {_table}" # noqa: S608
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def utc_iso(ts) -> str:
|
|
76
|
+
"""a timestamp as naive UTC ISO, the way the ABC stores tag dates"""
|
|
77
|
+
if isinstance(ts, datetime):
|
|
78
|
+
if ts.tzinfo is not None:
|
|
79
|
+
ts = ts.astimezone(timezone.utc).replace(tzinfo=None)
|
|
80
|
+
return ts.isoformat()
|
|
81
|
+
return str(ts) if ts is not None else ""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def valid_paper(data: CaltechData, joinkey: str) -> Optional[str]:
|
|
85
|
+
"""the valid paper a paper id resolves to through merges, or None"""
|
|
86
|
+
seen = set()
|
|
87
|
+
while joinkey not in data.valid:
|
|
88
|
+
if joinkey in seen or joinkey not in data.merged_into:
|
|
89
|
+
return None
|
|
90
|
+
seen.add(joinkey)
|
|
91
|
+
joinkey = data.merged_into[joinkey]
|
|
92
|
+
return joinkey
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _keep_rows(data: CaltechData, table_rows: List[tuple]) -> Dict[str, Row]:
|
|
96
|
+
"""rows by valid paper: the paper's own row wins over a merged paper's row, then the latest wins
|
|
97
|
+
(e.g. the pipeline rerun of a merged duplicate must not replace the run the author saw)"""
|
|
98
|
+
kept: Dict[str, Row] = {}
|
|
99
|
+
own: Set[str] = set()
|
|
100
|
+
for joinkey, value, ts in table_rows:
|
|
101
|
+
valid = valid_paper(data, joinkey)
|
|
102
|
+
if valid is None:
|
|
103
|
+
continue
|
|
104
|
+
if joinkey == valid:
|
|
105
|
+
own.add(valid)
|
|
106
|
+
elif valid in own:
|
|
107
|
+
continue
|
|
108
|
+
kept[valid] = Row(value or "", utc_iso(ts))
|
|
109
|
+
return kept
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _person(who: str) -> str:
|
|
113
|
+
return who.replace("two", "WBPerson", 1)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _pap_value(value: Optional[str]) -> str:
|
|
117
|
+
"""the id of a pap_gene/pap_species value: '00000830;%;ctl-1' and 'ctl-1 (WBGene00000830)' -> '00000830'"""
|
|
118
|
+
match = re.search(r"WBGene(\d{8})", value or "")
|
|
119
|
+
return match.group(1) if match else (value or "").split(";%;", 1)[0].strip()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _removals(data: CaltechData, history: List[tuple], current: List[tuple]) -> Dict[str, List[Removal]]:
|
|
123
|
+
"""the author entries a curator removed in the paper editor and that are not back in the pap_* table, by paper.
|
|
124
|
+
The paper editor logs a removal as an h_pap_* row with no value at the entry's pap_order; the entry is the
|
|
125
|
+
author's when an author row set its value (rows that only repeat the value keep it the author's). The latest
|
|
126
|
+
removal of a value wins. Papers that are no longer valid are skipped: their entries were emptied by a merge."""
|
|
127
|
+
present: Dict[str, Set[str]] = {}
|
|
128
|
+
for joinkey, value in current:
|
|
129
|
+
valid = valid_paper(data, joinkey)
|
|
130
|
+
if valid and _pap_value(value):
|
|
131
|
+
present.setdefault(valid, set()).add(_pap_value(value))
|
|
132
|
+
slots: Dict[tuple, tuple] = {} # (joinkey, pap_order) -> (value, set by an author row)
|
|
133
|
+
removed: Dict[str, Dict[str, Removal]] = {}
|
|
134
|
+
for joinkey, value, order, curator, evidence, ts in history:
|
|
135
|
+
value = _pap_value(value)
|
|
136
|
+
if value:
|
|
137
|
+
author = AUTHOR_EVIDENCE in (evidence or "")
|
|
138
|
+
old = slots.get((joinkey, order))
|
|
139
|
+
slots[(joinkey, order)] = (value, author or (old is not None and old == (value, True)))
|
|
140
|
+
continue
|
|
141
|
+
old = slots.pop((joinkey, order), None)
|
|
142
|
+
if old is None or not old[1] or joinkey not in data.valid or not curator \
|
|
143
|
+
or old[0] in present.get(joinkey, set()):
|
|
144
|
+
continue
|
|
145
|
+
removed.setdefault(joinkey, {})[old[0]] = Removal(old[0], _person(curator), utc_iso(ts))
|
|
146
|
+
return {paper: list(values.values()) for paper, values in removed.items()}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _species_name(obo_text: str) -> Optional[str]:
|
|
150
|
+
match = re.search(r'species: "(.*?)"', obo_text or "")
|
|
151
|
+
return match.group(1) if match else None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def build_data(rows: Dict[str, List[tuple]]) -> CaltechData:
|
|
155
|
+
"""CaltechData from the rows of QUERIES (keys missing from rows count as empty)"""
|
|
156
|
+
data = CaltechData()
|
|
157
|
+
data.valid = {r[0] for r in rows.get("valid", [])}
|
|
158
|
+
data.merged_into = {r[1]: r[0] for r in rows.get("merges", [])}
|
|
159
|
+
data.agrkb = {r[0]: r[1] for r in rows.get("agrkb", []) if r[0] in data.valid}
|
|
160
|
+
for joinkey, ts in rows.get("lasttouched", []):
|
|
161
|
+
valid = valid_paper(data, joinkey)
|
|
162
|
+
if valid and ts:
|
|
163
|
+
data.lasttouched[valid] = utc_iso(ts)
|
|
164
|
+
for joinkey, who, ts in rows.get("contributors_afp", []):
|
|
165
|
+
valid = valid_paper(data, joinkey)
|
|
166
|
+
# ACKnowledge saved "twoundefined" for a few submissions: no known person, so such a paper's tags go to
|
|
167
|
+
# unknown_author like those of a paper without a contributor
|
|
168
|
+
if who and valid and re.fullmatch(r"two\d+", who.strip()):
|
|
169
|
+
data.contributors.setdefault(valid, {})[_person(who)] = utc_iso(ts)
|
|
170
|
+
for table in ENTITY_TABLES + FLAG_TABLES:
|
|
171
|
+
data.tables[table] = _keep_rows(data, rows.get(table, []))
|
|
172
|
+
data.ack_papers = {joinkey for table in ACK_SECTION_TABLES for joinkey in data.tables.get(table, {})}
|
|
173
|
+
own_levels: Set[tuple] = set()
|
|
174
|
+
for paper, datatype, level, _ts in rows.get("blackbox", []): # own rows first, then the latest, as tables
|
|
175
|
+
valid = valid_paper(data, paper)
|
|
176
|
+
if not valid or not level:
|
|
177
|
+
continue
|
|
178
|
+
if paper == valid:
|
|
179
|
+
own_levels.add((valid, datatype))
|
|
180
|
+
elif (valid, datatype) in own_levels:
|
|
181
|
+
continue
|
|
182
|
+
data.blackbox.setdefault(valid, {})[datatype] = level.upper()
|
|
183
|
+
|
|
184
|
+
data.species_taxon = {name: "NCBITaxon:" + taxon_id for taxon_id, name, _ts in rows.get("species_index", [])}
|
|
185
|
+
data.gene_taxon = {"WB:WBGene" + gene: data.species_taxon[species] for gene, species in rows.get("genes", [])
|
|
186
|
+
if species in data.species_taxon}
|
|
187
|
+
for wbvar, obo_text in rows.get("variations", []):
|
|
188
|
+
species = _species_name(obo_text)
|
|
189
|
+
if species in data.species_taxon:
|
|
190
|
+
data.variation_taxon["WB:" + wbvar] = data.species_taxon[species]
|
|
191
|
+
for wbstrain, obo_text in rows.get("strains", []):
|
|
192
|
+
if not obo_text:
|
|
193
|
+
continue
|
|
194
|
+
names = re.findall(r'name: "(.*?)"', obo_text)
|
|
195
|
+
data.strain_by_name[names[0] if names else ""] = wbstrain
|
|
196
|
+
species = _species_name(obo_text)
|
|
197
|
+
if species in data.species_taxon:
|
|
198
|
+
data.strain_taxon["WB:" + wbstrain] = data.species_taxon[species]
|
|
199
|
+
data.strain_by_name.pop("", None)
|
|
200
|
+
for wbtransgene, public_name, species in rows.get("transgenes", []):
|
|
201
|
+
if public_name:
|
|
202
|
+
data.transgene_by_name[public_name] = wbtransgene
|
|
203
|
+
if species in data.species_taxon:
|
|
204
|
+
data.transgene_taxon["WB:" + wbtransgene] = data.species_taxon[species]
|
|
205
|
+
for name in PAPER_EDITOR_TABLES:
|
|
206
|
+
data.removals[name] = _removals(data, rows.get(f"history_{name}", []), rows.get(f"current_{name}", []))
|
|
207
|
+
return data
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def load(conn) -> CaltechData:
|
|
211
|
+
"""run QUERIES on the Caltech database in one read-only snapshot (a lookup table rebuilt during the run would
|
|
212
|
+
otherwise make entities unresolvable halfway)"""
|
|
213
|
+
conn.set_session(readonly=True, isolation_level="REPEATABLE READ")
|
|
214
|
+
cur = conn.cursor()
|
|
215
|
+
rows: Dict[str, List[tuple]] = {}
|
|
216
|
+
for key, sql in QUERIES.items():
|
|
217
|
+
cur.execute(sql)
|
|
218
|
+
rows[key] = cur.fetchall()
|
|
219
|
+
cur.close()
|
|
220
|
+
return build_data(rows)
|