wbtools 3.7.2__tar.gz → 3.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. {wbtools-3.7.2/wbtools.egg-info → wbtools-3.8.0}/PKG-INFO +3 -2
  2. {wbtools-3.7.2 → wbtools-3.8.0}/pyproject.toml +1 -1
  3. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/dbmanager.py +16 -0
  4. wbtools-3.8.0/wbtools/literature/ack_form/__init__.py +1 -0
  5. wbtools-3.8.0/wbtools/literature/ack_form/abc.py +82 -0
  6. wbtools-3.8.0/wbtools/literature/ack_form/author.py +183 -0
  7. wbtools-3.8.0/wbtools/literature/ack_form/caltech.py +220 -0
  8. wbtools-3.8.0/wbtools/literature/ack_form/changes.py +147 -0
  9. wbtools-3.8.0/wbtools/literature/ack_form/constants.py +108 -0
  10. wbtools-3.8.0/wbtools/literature/ack_form/entities.py +92 -0
  11. wbtools-3.8.0/wbtools/literature/ack_form/paper_data.py +80 -0
  12. wbtools-3.8.0/wbtools/literature/ack_form/reverse.py +134 -0
  13. wbtools-3.8.0/wbtools/literature/ack_form/tags.py +46 -0
  14. {wbtools-3.7.2 → wbtools-3.8.0/wbtools.egg-info}/PKG-INFO +3 -2
  15. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools.egg-info/SOURCES.txt +10 -0
  16. {wbtools-3.7.2 → wbtools-3.8.0}/LICENSE +0 -0
  17. {wbtools-3.7.2 → wbtools-3.8.0}/README.md +0 -0
  18. {wbtools-3.7.2 → wbtools-3.8.0}/setup.cfg +0 -0
  19. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/__init__.py +0 -0
  20. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/__init__.py +0 -0
  21. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/abstract_manager.py +0 -0
  22. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/afp.py +0 -0
  23. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/antibody.py +0 -0
  24. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/expression.py +0 -0
  25. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/gene.py +0 -0
  26. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/generic.py +0 -0
  27. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/paper.py +0 -0
  28. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/db/person.py +0 -0
  29. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/__init__.py +0 -0
  30. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/email/__init__.py +0 -0
  31. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/email/generic.py +0 -0
  32. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/__init__.py +0 -0
  33. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/common.py +0 -0
  34. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/entity_extraction/__init__.py +0 -0
  35. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/entity_extraction/email_addresses.py +0 -0
  36. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/entity_extraction/ntt_extractor.py +0 -0
  37. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/entity_extraction/variations.py +0 -0
  38. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/literature_index/__init__.py +0 -0
  39. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/literature_index/abstract_index.py +0 -0
  40. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/literature_index/textpresso.py +0 -0
  41. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/text_preprocessing.py +0 -0
  42. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/nlp/text_similarity.py +0 -0
  43. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/scraping.py +0 -0
  44. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/lib/timeout.py +0 -0
  45. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/literature/__init__.py +0 -0
  46. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/literature/abc_entities.py +0 -0
  47. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/literature/abc_search.py +0 -0
  48. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/literature/abc_tag_writer.py +0 -0
  49. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/literature/abc_tags.py +0 -0
  50. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/literature/corpus.py +0 -0
  51. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/literature/paper.py +0 -0
  52. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/literature/person.py +0 -0
  53. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/utils/__init__.py +0 -0
  54. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/utils/alliance.py +0 -0
  55. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools/utils/auth_utils.py +0 -0
  56. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools.egg-info/dependency_links.txt +0 -0
  57. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools.egg-info/requires.txt +0 -0
  58. {wbtools-3.7.2 → wbtools-3.8.0}/wbtools.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: wbtools
3
- Version: 3.7.2
3
+ Version: 3.8.0
4
4
  Summary: Interface to WormBase (www.wormbase.org) curation data, including literature management and NLP functions
5
5
  Author-email: Valerio Arnaboldi <valearna@caltech.edu>
6
6
  Project-URL: Homepage, https://github.com/WormBase/wbtools
@@ -26,6 +26,7 @@ Requires-Dist: python-dotenv~=1.0.0
26
26
  Requires-Dist: grobid-client
27
27
  Requires-Dist: agr-cognito-py
28
28
  Requires-Dist: agr-abc-document-parsers>=1.7.2
29
+ Dynamic: license-file
29
30
 
30
31
  # WBtools
31
32
  > Interface to WormBase curation database and Text Mining functions
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "wbtools"
7
- version = "3.7.2"
7
+ version = "3.8.0"
8
8
  authors = [
9
9
  {name = "Valerio Arnaboldi", email = "valearna@caltech.edu"},
10
10
  ]
@@ -1,4 +1,5 @@
1
1
  import logging
2
+ from contextlib import contextmanager
2
3
 
3
4
  import psycopg2
4
5
 
@@ -46,3 +47,18 @@ class WBDBManager(AbstractWBDBManager):
46
47
 
47
48
  def __exit__(self, exc_type, exc_val, exc_tb):
48
49
  super(WBDBManager, self).__exit__(exc_type, exc_val, exc_tb)
50
+
51
+ @contextmanager
52
+ def transaction(self):
53
+ """one transaction for every sub-manager: committed on success, rolled back on any error (unlike
54
+ ``with manager:``, which commits even when the block raised), and the connection always closed"""
55
+ self.__enter__()
56
+ try:
57
+ yield self
58
+ self.conn.commit()
59
+ except BaseException:
60
+ self.conn.rollback()
61
+ raise
62
+ finally:
63
+ self.curs.close()
64
+ self.conn.close()
@@ -0,0 +1 @@
1
+ """ACKnowledge_form tags: the author answers of the ACKnowledge form as ABC topic entity tags (SCRUM-4141)."""
@@ -0,0 +1,82 @@
1
+ """The ABC calls of an ACKnowledge save: read a paper's ACKnowledge_form tags, send the save's changes as one batch
2
+ (POST /topic_entity_tag/batch, SCRUM-4141)."""
3
+ from typing import Dict, List
4
+
5
+ import requests
6
+
7
+ from wbtools.literature.ack_form.changes import TagBatch
8
+ from wbtools.literature.ack_form.constants import ACK_FORM, SOURCES
9
+ from wbtools.literature.paper import ABC_API, ABCRequestError
10
+ from wbtools.utils.auth_utils import generate_headers, get_authentication_token
11
+
12
+ # the ABC refuses larger batches; refusing here saves the call
13
+ MAX_BATCH_CHANGES = 5000
14
+ _SOURCE_IDS: Dict[str, int] = {}
15
+
16
+
17
+ def _post(path: str, body: dict, timeout: int) -> requests.Response:
18
+ headers = generate_headers(get_authentication_token())
19
+ try:
20
+ return requests.post(f"https://{ABC_API}{path}", json=body, headers=headers, timeout=timeout)
21
+ except requests.exceptions.RequestException as e:
22
+ raise ABCRequestError(f"ABC {path} failed: {e}") from e
23
+
24
+
25
+ def get_ack_form_source_id(timeout: int) -> int:
26
+ """the id of the WB ACKnowledge_form tag source, looked up once per process
27
+
28
+ Raises:
29
+ ABCRequestError: if the ABC can't be read
30
+ """
31
+ if ACK_FORM not in _SOURCE_IDS:
32
+ path = f"/tag_source/{SOURCES[ACK_FORM]}/{ACK_FORM}/WB/WB"
33
+ try:
34
+ response = requests.get(f"https://{ABC_API}{path}",
35
+ headers=generate_headers(get_authentication_token()), timeout=timeout)
36
+ except requests.exceptions.RequestException as e:
37
+ raise ABCRequestError(f"ABC {path} failed: {e}") from e
38
+ if response.status_code != 200:
39
+ raise ABCRequestError(f"ABC {path} failed: HTTP {response.status_code} {response.text[:200]}")
40
+ _SOURCE_IDS[ACK_FORM] = response.json()["tag_source_id"]
41
+ return _SOURCE_IDS[ACK_FORM]
42
+
43
+
44
+ def get_ack_form_tags(agr_curie: str, timeout: int) -> List[dict]:
45
+ """the paper's ACKnowledge_form tags, every contributor's copy
46
+
47
+ Raises:
48
+ ABCRequestError: if the ABC can't be read
49
+ """
50
+ response = _post("/topic_entity_tag/by_references", {
51
+ "curies_or_reference_ids": [agr_curie], "filters": {"source_methods": [ACK_FORM]}}, timeout)
52
+ if response.status_code != 200:
53
+ raise ABCRequestError(f"ABC by_references failed for {agr_curie}: HTTP {response.status_code} "
54
+ f"{response.text[:500]}")
55
+ try:
56
+ tags = response.json()["tags"]
57
+ except (ValueError, KeyError, TypeError) as e:
58
+ raise ABCRequestError(f"ABC by_references gave an unexpected answer for {agr_curie}: {e}") from e
59
+ return tags.get(agr_curie) or []
60
+
61
+
62
+ def send_tag_batch(agr_curie: str, batch: TagBatch, timeout: int) -> List[int]:
63
+ """apply a save's tag changes in one ABC transaction; returns the ids of the created tags
64
+
65
+ Raises:
66
+ ABCRequestError: if the batch is over the cap or the ABC refuses it or can't be reached (the ABC then stored
67
+ nothing, unless the answer was lost after the ABC committed)
68
+ """
69
+ if batch.size() > MAX_BATCH_CHANGES:
70
+ raise ABCRequestError(f"{batch.size()} tag changes for {agr_curie}; the ABC accepts at most "
71
+ f"{MAX_BATCH_CHANGES} in one batch")
72
+ body = {"reference_curie": agr_curie,
73
+ "create": [dict(tag, reference_curie=agr_curie) for tag in batch.create],
74
+ "update": batch.update, "delete": batch.delete}
75
+ response = _post("/topic_entity_tag/batch", body, timeout)
76
+ if response.status_code != 200:
77
+ raise ABCRequestError(f"ABC tag batch refused for {agr_curie}: HTTP {response.status_code} "
78
+ f"{response.text[:1000]}")
79
+ try:
80
+ return response.json()["created"]
81
+ except (ValueError, KeyError, TypeError) as e:
82
+ raise ABCRequestError(f"ABC tag batch gave an unexpected answer for {agr_curie}: {e}") from e
@@ -0,0 +1,183 @@
1
+ """The ACKnowledge_form tags of a paper's ACKnowledge submission, final or partial (spec amendment 12)."""
2
+ import json
3
+ import re
4
+ from typing import List, Optional, Tuple
5
+
6
+ from wbtools.literature.ack_form.caltech import CaltechData
7
+ from wbtools.literature.ack_form.constants import (ACK_FORM, ACK_START, DATATYPE_FLAGS, EMPTY_OTHER,
8
+ ENTITY_TYPES, NOVELTY_EXISTING, NOVELTY_NEW_TO_DB)
9
+ from wbtools.literature.ack_form.entities import other_names, other_note, parse_entities
10
+ from wbtools.literature.ack_form.tags import author_key, make_tag
11
+
12
+ AUTHOR_TABLES = [table for entity_type in ENTITY_TYPES.values()
13
+ for table in (entity_type.afp_table, entity_type.other_table) if table] + \
14
+ [flag.table for flag in DATATYPE_FLAGS]
15
+
16
+
17
+ def _latest_submission(joinkey: str, data: CaltechData) -> str:
18
+ timestamps = [data.tables.get(table, {})[joinkey].timestamp for table in AUTHOR_TABLES
19
+ if joinkey in data.tables.get(table, {})]
20
+ timestamps.append(data.lasttouched.get(joinkey, ""))
21
+ return max(timestamps)
22
+
23
+
24
+ def in_author_scope(joinkey: str, data: CaltechData) -> bool:
25
+ """an ACKnowledge submission, final or partial (old author first pass submissions are out of scope)"""
26
+ return joinkey in data.ack_papers and _latest_submission(joinkey, data)[:10] >= ACK_START
27
+
28
+
29
+ def authors(joinkey: str, data: CaltechData) -> List[Tuple[str, Optional[str]]]:
30
+ """(WBPerson, contribution timestamp), sorted; unknown_author when the paper has no contributor"""
31
+ contributors = data.contributors.get(joinkey) or {}
32
+ return sorted(contributors.items()) or [("unknown_author", None)]
33
+
34
+
35
+ def _date(contributed: Optional[str], row_ts: Optional[str], joinkey: str, data: CaltechData) -> Optional[str]:
36
+ return contributed or row_ts or data.lasttouched.get(joinkey) or None
37
+
38
+
39
+ def _note(entities) -> str:
40
+ return "\n".join(f"{e.published_as} ( {e.curie} )" if e.published_as else e.curie for e in entities)
41
+
42
+
43
+ def author_entity_tags(joinkey: str, curie: str, data: CaltechData) -> Tuple[List[dict], List[Tuple[str, str]]]:
44
+ """the expected ACKnowledge_form entity tags of a paper, and the problems found as (topic, message)"""
45
+ if not in_author_scope(joinkey, data):
46
+ return [], []
47
+ tags: List[dict] = []
48
+ problems: List[Tuple[str, str]] = []
49
+ paper_authors = authors(joinkey, data)
50
+ for entity_type in ENTITY_TYPES.values():
51
+ afp = data.tables.get(entity_type.afp_table, {}).get(joinkey)
52
+ other = data.tables.get(entity_type.other_table, {}).get(joinkey) if entity_type.other_table else None
53
+ tfp = data.tables.get(entity_type.tfp_table, {}).get(joinkey)
54
+ afp_entities, removed = [], []
55
+ if afp is not None and afp.data:
56
+ afp_entities, afp_problems = parse_entities(entity_type.name, afp.data, data)
57
+ problems.extend((entity_type.topic, f"{entity_type.afp_table}: {p}") for p in afp_problems)
58
+ # an unresolved author item could be any of the pipeline entities: no "removed" negations then
59
+ if tfp is not None and tfp.data and not afp_problems:
60
+ kept = {e.curie for e in afp_entities}
61
+ removed = [e for e in parse_entities(entity_type.name, tfp.data, data)[0] if e.curie not in kept]
62
+ names = other_names(other.data) if other is not None else []
63
+ for who, contributed in paper_authors:
64
+ def tag(row_ts, **fields):
65
+ tags.append(make_tag(reference_curie=curie, source_method=ACK_FORM, topic=entity_type.topic,
66
+ created_by=who, date_created=_date(contributed, row_ts, joinkey, data),
67
+ **fields))
68
+
69
+ for entity in afp_entities:
70
+ tag(afp.timestamp, entity_type=entity_type.topic, entity=entity.curie, species=entity.species,
71
+ entity_id_validation="alliance", negated=False, data_novelty=NOVELTY_EXISTING,
72
+ note=_note(afp_entities))
73
+ for entity in removed:
74
+ tag(afp.timestamp, entity_type=entity_type.topic, entity=entity.curie, species=entity.species,
75
+ entity_id_validation="alliance", negated=True, data_novelty=NOVELTY_EXISTING)
76
+ if afp is not None and afp.data == "":
77
+ tag(afp.timestamp, negated=True, data_novelty=NOVELTY_EXISTING)
78
+ if names:
79
+ tag(other.timestamp, negated=False, data_novelty=NOVELTY_NEW_TO_DB,
80
+ note=other_note(other.data) or "\n".join(names))
81
+ elif other is not None and other.data == EMPTY_OTHER:
82
+ tag(other.timestamp, negated=True, data_novelty=NOVELTY_NEW_TO_DB)
83
+ return tags, problems
84
+
85
+
86
+ def _otherantibody(value: str) -> Tuple[bool, str]:
87
+ try:
88
+ entries = json.loads(value or "")
89
+ except ValueError:
90
+ return True, ""
91
+ if not isinstance(entries, list):
92
+ return True, ""
93
+ parts = []
94
+ for entry in entries:
95
+ if not isinstance(entry, dict):
96
+ continue
97
+ fields = [str(entry[key]) for key in ("name", "publicationId") if entry.get(key) not in (None, "")]
98
+ if fields:
99
+ parts.append("; ".join(fields))
100
+ note = " | ".join(parts)
101
+ return note == "", note
102
+
103
+
104
+ def _humandisease(value: Optional[str]) -> Tuple[bool, str]:
105
+ if not value:
106
+ return True, ""
107
+ try:
108
+ parsed = json.loads(value)
109
+ except ValueError:
110
+ return False, _free_text(value)
111
+ if not isinstance(parsed, dict):
112
+ return False, _free_text(value)
113
+ # SCRUM-4141: the first line is the comment (empty when there is none), each further line a disease, so the
114
+ # answer can be rebuilt from the note (reverse.py). The author's own newlines become spaces: a newline in the
115
+ # note is only ever a separator.
116
+ comment = parsed.get("comment")
117
+ comment = "" if comment in (None, "", "checked") else _one_line(str(comment))
118
+ diseases = []
119
+ for disease in parsed.get("diseases") or []:
120
+ disease = re.sub(r"\s+\)", ")", re.sub(r"\(\s+", "(", _one_line(str(disease))))
121
+ if disease:
122
+ diseases.append(disease)
123
+ return not parsed.get("checked"), "\n".join([comment] + diseases) if diseases else comment
124
+
125
+
126
+ def _free_text(value: str) -> str:
127
+ """a disease answer that is not the form's JSON, on one line as every other flag note"""
128
+ note = value.replace("\n", " ")
129
+ return note[:-1] if note.endswith(" ") else note
130
+
131
+
132
+ def _one_line(text: str) -> str:
133
+ return text.replace("\r", " ").replace("\n", " ").rstrip()
134
+
135
+
136
+ def convert_flag(datatype: str, value: Optional[str]) -> Tuple[bool, str]:
137
+ """(negated, note) of an afp_<datatype> value, as convertAckToNote in populate_classifier_topic_entity.pl"""
138
+ if datatype == "otherantibody":
139
+ negated, note = _otherantibody(value or "")
140
+ elif datatype == "humandisease":
141
+ return _humandisease(value) # its newlines are separators; its text is already on one line
142
+ else:
143
+ negated = value in ("", None, "0")
144
+ note = "" if negated else value
145
+ note = note.replace("\n", " ")
146
+ return negated, note[:-1] if note.endswith(" ") else note
147
+
148
+
149
+ def author_flag_tags(joinkey: str, curie: str, data: CaltechData) -> List[dict]:
150
+ """the expected ACKnowledge_form data-type flag tags of a paper: rows dated from ACK_START on, as
151
+ populate_classifier_topic_entity.pl decides (early ACKnowledge submissions have no entity sections, so the
152
+ ACKnowledge section test of the entity tags does not apply here)"""
153
+ tags = []
154
+ for flag in DATATYPE_FLAGS:
155
+ row = data.tables.get(flag.table, {}).get(joinkey)
156
+ if row is None or row.timestamp[:10] < ACK_START:
157
+ continue
158
+ negated, note = convert_flag(flag.datatype, row.data)
159
+ for who, contributed in authors(joinkey, data):
160
+ tags.append(make_tag(reference_curie=curie, source_method=ACK_FORM, topic=flag.topic, negated=negated,
161
+ data_novelty=flag.novelty, note=note or None, created_by=who,
162
+ date_created=_date(contributed, row.timestamp, joinkey, data)))
163
+ return _merge_notes(tags)
164
+
165
+
166
+ def _merge_notes(tags: List[dict]) -> List[dict]:
167
+ """two flags of one topic (marker and otherexpr) give tags that differ only in their note; the ABC stores those
168
+ as one tag, the first one created, with the notes joined by " | " (create_tag appends the note), so the
169
+ expected tags are merged the same way"""
170
+ merged: dict = {}
171
+ for tag in tags:
172
+ first = merged.setdefault(author_key({**tag, "note": None}), tag)
173
+ if first is not tag and tag.get("note"):
174
+ notes = first["note"].split(" | ") if first.get("note") else []
175
+ if tag["note"] not in notes:
176
+ first["note"] = " | ".join(notes + [tag["note"]])
177
+ return list(merged.values())
178
+
179
+
180
+ def author_tags(joinkey: str, curie: str, data: CaltechData) -> Tuple[List[dict], List[Tuple[str, str]]]:
181
+ """all the expected ACKnowledge_form tags of a paper (entities, then data-type flags), and the problems found"""
182
+ entity_tags, problems = author_entity_tags(joinkey, curie, data)
183
+ return entity_tags + author_flag_tags(joinkey, curie, data), problems
@@ -0,0 +1,220 @@
1
+ """Read the Caltech tables the sync needs into a CaltechData (read only)."""
2
+ import re
3
+ from dataclasses import dataclass, field
4
+ from datetime import datetime, timezone
5
+ from typing import Dict, List, NamedTuple, Optional, Set
6
+
7
+ from wbtools.literature.ack_form.constants import (ACK_SECTION_TABLES, AUTHOR_EVIDENCE, DATATYPE_FLAGS,
8
+ ENTITY_TYPES, PAPER_EDITOR_TABLES)
9
+
10
+
11
+ class Row(NamedTuple):
12
+ data: str
13
+ timestamp: str
14
+
15
+
16
+ class Removal(NamedTuple):
17
+ """an author's paper editor entry (pap_gene/pap_species "from author first pass") a curator removed"""
18
+ value: str # the pap_* value: an 8-digit gene number or an NCBI taxon id
19
+ curator: str # WBPerson
20
+ timestamp: str
21
+
22
+
23
+ @dataclass
24
+ class CaltechData:
25
+ valid: Set[str] = field(default_factory=set)
26
+ merged_into: Dict[str, str] = field(default_factory=dict) # merged paper -> paper it was merged into
27
+ agrkb: Dict[str, str] = field(default_factory=dict)
28
+ lasttouched: Dict[str, str] = field(default_factory=dict) # final ACKnowledge submissions
29
+ ack_papers: Set[str] = field(default_factory=set)
30
+ contributors: Dict[str, Dict[str, str]] = field(default_factory=dict) # paper -> WBPerson -> timestamp
31
+ tables: Dict[str, Dict[str, Row]] = field(default_factory=dict) # tfp_*/afp_* table -> paper -> row
32
+ blackbox: Dict[str, Dict[str, str]] = field(default_factory=dict) # paper -> datatype -> level
33
+ gene_taxon: Dict[str, str] = field(default_factory=dict)
34
+ variation_taxon: Dict[str, str] = field(default_factory=dict)
35
+ strain_by_name: Dict[str, str] = field(default_factory=dict)
36
+ strain_taxon: Dict[str, str] = field(default_factory=dict)
37
+ transgene_by_name: Dict[str, str] = field(default_factory=dict)
38
+ transgene_taxon: Dict[str, str] = field(default_factory=dict)
39
+ species_taxon: Dict[str, str] = field(default_factory=dict) # species name -> NCBITaxon curie
40
+ removals: Dict[str, Dict[str, List[Removal]]] = field(default_factory=dict) # gene/species -> paper -> removals
41
+
42
+
43
+ ENTITY_TABLES = [table for entity_type in ENTITY_TYPES.values()
44
+ for table in (entity_type.tfp_table, entity_type.afp_table, entity_type.other_table) if table]
45
+ FLAG_TABLES = [flag.table for flag in DATATYPE_FLAGS]
46
+
47
+
48
+ def _table_query(table: str) -> str:
49
+ prefix = table.split("_")[0]
50
+ return f"SELECT joinkey, {table}, {prefix}_timestamp FROM {table} ORDER BY {prefix}_timestamp" # noqa: S608
51
+
52
+
53
+ QUERIES: Dict[str, str] = {
54
+ "valid": "SELECT joinkey FROM pap_status WHERE pap_status = 'valid'",
55
+ "merges": "SELECT joinkey, pap_identifier FROM pap_identifier WHERE pap_identifier ~ '^[0-9]{8}$'",
56
+ "agrkb": "SELECT joinkey, pap_identifier FROM pap_identifier WHERE pap_identifier ~ 'AGRKB'",
57
+ "lasttouched": "SELECT joinkey, afp_timestamp FROM afp_lasttouched ORDER BY afp_timestamp",
58
+ # the people who contributed to the submission (not pap_curator, which names only the one who submitted)
59
+ "contributors_afp": "SELECT joinkey, afp_contributor, afp_timestamp FROM afp_contributor ORDER BY afp_timestamp",
60
+ "blackbox": "SELECT cur_paper, cur_datatype, cur_blackbox, cur_timestamp FROM cur_blackbox ORDER BY cur_timestamp",
61
+ "species_index": "SELECT joinkey, pap_species_index, pap_timestamp FROM pap_species_index ORDER BY pap_timestamp",
62
+ "genes": "SELECT joinkey, gin_species FROM gin_species",
63
+ "variations": "SELECT joinkey, obo_data_variation FROM obo_data_variation",
64
+ "strains": "SELECT joinkey, obo_data_strain FROM obo_data_strain",
65
+ "transgenes": "SELECT n.trp_name, p.trp_publicname, s.trp_species FROM trp_name n "
66
+ "LEFT JOIN trp_publicname p USING (joinkey) LEFT JOIN trp_species s USING (joinkey)",
67
+ }
68
+ QUERIES.update({table: _table_query(table) for table in ENTITY_TABLES + FLAG_TABLES})
69
+ for _name, _table in PAPER_EDITOR_TABLES.items():
70
+ QUERIES[f"history_{_name}"] = (f"SELECT joinkey, {_table}, pap_order, pap_curator, pap_evidence, pap_timestamp "
71
+ f"FROM h_{_table} ORDER BY pap_timestamp") # noqa: S608
72
+ QUERIES[f"current_{_name}"] = f"SELECT joinkey, {_table} FROM {_table}" # noqa: S608
73
+
74
+
75
+ def utc_iso(ts) -> str:
76
+ """a timestamp as naive UTC ISO, the way the ABC stores tag dates"""
77
+ if isinstance(ts, datetime):
78
+ if ts.tzinfo is not None:
79
+ ts = ts.astimezone(timezone.utc).replace(tzinfo=None)
80
+ return ts.isoformat()
81
+ return str(ts) if ts is not None else ""
82
+
83
+
84
+ def valid_paper(data: CaltechData, joinkey: str) -> Optional[str]:
85
+ """the valid paper a paper id resolves to through merges, or None"""
86
+ seen = set()
87
+ while joinkey not in data.valid:
88
+ if joinkey in seen or joinkey not in data.merged_into:
89
+ return None
90
+ seen.add(joinkey)
91
+ joinkey = data.merged_into[joinkey]
92
+ return joinkey
93
+
94
+
95
+ def _keep_rows(data: CaltechData, table_rows: List[tuple]) -> Dict[str, Row]:
96
+ """rows by valid paper: the paper's own row wins over a merged paper's row, then the latest wins
97
+ (e.g. the pipeline rerun of a merged duplicate must not replace the run the author saw)"""
98
+ kept: Dict[str, Row] = {}
99
+ own: Set[str] = set()
100
+ for joinkey, value, ts in table_rows:
101
+ valid = valid_paper(data, joinkey)
102
+ if valid is None:
103
+ continue
104
+ if joinkey == valid:
105
+ own.add(valid)
106
+ elif valid in own:
107
+ continue
108
+ kept[valid] = Row(value or "", utc_iso(ts))
109
+ return kept
110
+
111
+
112
+ def _person(who: str) -> str:
113
+ return who.replace("two", "WBPerson", 1)
114
+
115
+
116
+ def _pap_value(value: Optional[str]) -> str:
117
+ """the id of a pap_gene/pap_species value: '00000830;%;ctl-1' and 'ctl-1 (WBGene00000830)' -> '00000830'"""
118
+ match = re.search(r"WBGene(\d{8})", value or "")
119
+ return match.group(1) if match else (value or "").split(";%;", 1)[0].strip()
120
+
121
+
122
+ def _removals(data: CaltechData, history: List[tuple], current: List[tuple]) -> Dict[str, List[Removal]]:
123
+ """the author entries a curator removed in the paper editor and that are not back in the pap_* table, by paper.
124
+ The paper editor logs a removal as an h_pap_* row with no value at the entry's pap_order; the entry is the
125
+ author's when an author row set its value (rows that only repeat the value keep it the author's). The latest
126
+ removal of a value wins. Papers that are no longer valid are skipped: their entries were emptied by a merge."""
127
+ present: Dict[str, Set[str]] = {}
128
+ for joinkey, value in current:
129
+ valid = valid_paper(data, joinkey)
130
+ if valid and _pap_value(value):
131
+ present.setdefault(valid, set()).add(_pap_value(value))
132
+ slots: Dict[tuple, tuple] = {} # (joinkey, pap_order) -> (value, set by an author row)
133
+ removed: Dict[str, Dict[str, Removal]] = {}
134
+ for joinkey, value, order, curator, evidence, ts in history:
135
+ value = _pap_value(value)
136
+ if value:
137
+ author = AUTHOR_EVIDENCE in (evidence or "")
138
+ old = slots.get((joinkey, order))
139
+ slots[(joinkey, order)] = (value, author or (old is not None and old == (value, True)))
140
+ continue
141
+ old = slots.pop((joinkey, order), None)
142
+ if old is None or not old[1] or joinkey not in data.valid or not curator \
143
+ or old[0] in present.get(joinkey, set()):
144
+ continue
145
+ removed.setdefault(joinkey, {})[old[0]] = Removal(old[0], _person(curator), utc_iso(ts))
146
+ return {paper: list(values.values()) for paper, values in removed.items()}
147
+
148
+
149
+ def _species_name(obo_text: str) -> Optional[str]:
150
+ match = re.search(r'species: "(.*?)"', obo_text or "")
151
+ return match.group(1) if match else None
152
+
153
+
154
+ def build_data(rows: Dict[str, List[tuple]]) -> CaltechData:
155
+ """CaltechData from the rows of QUERIES (keys missing from rows count as empty)"""
156
+ data = CaltechData()
157
+ data.valid = {r[0] for r in rows.get("valid", [])}
158
+ data.merged_into = {r[1]: r[0] for r in rows.get("merges", [])}
159
+ data.agrkb = {r[0]: r[1] for r in rows.get("agrkb", []) if r[0] in data.valid}
160
+ for joinkey, ts in rows.get("lasttouched", []):
161
+ valid = valid_paper(data, joinkey)
162
+ if valid and ts:
163
+ data.lasttouched[valid] = utc_iso(ts)
164
+ for joinkey, who, ts in rows.get("contributors_afp", []):
165
+ valid = valid_paper(data, joinkey)
166
+ # ACKnowledge saved "twoundefined" for a few submissions: no known person, so such a paper's tags go to
167
+ # unknown_author like those of a paper without a contributor
168
+ if who and valid and re.fullmatch(r"two\d+", who.strip()):
169
+ data.contributors.setdefault(valid, {})[_person(who)] = utc_iso(ts)
170
+ for table in ENTITY_TABLES + FLAG_TABLES:
171
+ data.tables[table] = _keep_rows(data, rows.get(table, []))
172
+ data.ack_papers = {joinkey for table in ACK_SECTION_TABLES for joinkey in data.tables.get(table, {})}
173
+ own_levels: Set[tuple] = set()
174
+ for paper, datatype, level, _ts in rows.get("blackbox", []): # own rows first, then the latest, as tables
175
+ valid = valid_paper(data, paper)
176
+ if not valid or not level:
177
+ continue
178
+ if paper == valid:
179
+ own_levels.add((valid, datatype))
180
+ elif (valid, datatype) in own_levels:
181
+ continue
182
+ data.blackbox.setdefault(valid, {})[datatype] = level.upper()
183
+
184
+ data.species_taxon = {name: "NCBITaxon:" + taxon_id for taxon_id, name, _ts in rows.get("species_index", [])}
185
+ data.gene_taxon = {"WB:WBGene" + gene: data.species_taxon[species] for gene, species in rows.get("genes", [])
186
+ if species in data.species_taxon}
187
+ for wbvar, obo_text in rows.get("variations", []):
188
+ species = _species_name(obo_text)
189
+ if species in data.species_taxon:
190
+ data.variation_taxon["WB:" + wbvar] = data.species_taxon[species]
191
+ for wbstrain, obo_text in rows.get("strains", []):
192
+ if not obo_text:
193
+ continue
194
+ names = re.findall(r'name: "(.*?)"', obo_text)
195
+ data.strain_by_name[names[0] if names else ""] = wbstrain
196
+ species = _species_name(obo_text)
197
+ if species in data.species_taxon:
198
+ data.strain_taxon["WB:" + wbstrain] = data.species_taxon[species]
199
+ data.strain_by_name.pop("", None)
200
+ for wbtransgene, public_name, species in rows.get("transgenes", []):
201
+ if public_name:
202
+ data.transgene_by_name[public_name] = wbtransgene
203
+ if species in data.species_taxon:
204
+ data.transgene_taxon["WB:" + wbtransgene] = data.species_taxon[species]
205
+ for name in PAPER_EDITOR_TABLES:
206
+ data.removals[name] = _removals(data, rows.get(f"history_{name}", []), rows.get(f"current_{name}", []))
207
+ return data
208
+
209
+
210
+ def load(conn) -> CaltechData:
211
+ """run QUERIES on the Caltech database in one read-only snapshot (a lookup table rebuilt during the run would
212
+ otherwise make entities unresolvable halfway)"""
213
+ conn.set_session(readonly=True, isolation_level="REPEATABLE READ")
214
+ cur = conn.cursor()
215
+ rows: Dict[str, List[tuple]] = {}
216
+ for key, sql in QUERIES.items():
217
+ cur.execute(sql)
218
+ rows[key] = cur.fetchall()
219
+ cur.close()
220
+ return build_data(rows)