cdot 0.2.30__tar.gz → 0.2.32__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. {cdot-0.2.30 → cdot-0.2.32}/PKG-INFO +56 -3
  2. {cdot-0.2.30 → cdot-0.2.32}/README.md +55 -2
  3. {cdot-0.2.30 → cdot-0.2.32}/cdot/__init__.py +1 -1
  4. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/__init__.py +2 -0
  5. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/clean.py +76 -5
  6. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/dataproviders/json_data_provider.py +58 -20
  7. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/gene_hgvs.py +112 -3
  8. cdot-0.2.32/cdot/hgvs/translation.py +267 -0
  9. {cdot-0.2.30 → cdot-0.2.32}/cdot/models.py +57 -2
  10. {cdot-0.2.30 → cdot-0.2.32}/cdot.egg-info/PKG-INFO +56 -3
  11. {cdot-0.2.30 → cdot-0.2.32}/cdot.egg-info/SOURCES.txt +3 -0
  12. {cdot-0.2.30 → cdot-0.2.32}/tests/test_clean_hgvs.py +43 -0
  13. {cdot-0.2.30 → cdot-0.2.32}/tests/test_gene_hgvs.py +116 -3
  14. cdot-0.2.32/tests/test_gff_parsers.py +457 -0
  15. {cdot-0.2.30 → cdot-0.2.32}/tests/test_models.py +43 -0
  16. cdot-0.2.32/tests/test_retrieve_many.py +75 -0
  17. cdot-0.2.32/tests/test_translation.py +148 -0
  18. cdot-0.2.32/tests/test_uta_conversion.py +93 -0
  19. cdot-0.2.30/tests/test_gff_parsers.py +0 -188
  20. cdot-0.2.30/tests/test_uta_conversion.py +0 -43
  21. {cdot-0.2.30 → cdot-0.2.32}/LICENSE +0 -0
  22. {cdot-0.2.30 → cdot-0.2.32}/cdot/data_release.py +0 -0
  23. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/_version_replacement_blocklist.py +0 -0
  24. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/dataproviders/__init__.py +0 -0
  25. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/dataproviders/ensembl_tark_data_provider.py +0 -0
  26. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/dataproviders/fasta_seqfetcher.py +0 -0
  27. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/dataproviders/seqfetcher.py +0 -0
  28. {cdot-0.2.30 → cdot-0.2.32}/cdot/hgvs/version_safety.py +0 -0
  29. {cdot-0.2.30 → cdot-0.2.32}/cdot/pyhgvs/__init__.py +0 -0
  30. {cdot-0.2.30 → cdot-0.2.32}/cdot/pyhgvs/pyhgvs_transcript.py +0 -0
  31. {cdot-0.2.30 → cdot-0.2.32}/cdot.egg-info/dependency_links.txt +0 -0
  32. {cdot-0.2.30 → cdot-0.2.32}/cdot.egg-info/requires.txt +0 -0
  33. {cdot-0.2.30 → cdot-0.2.32}/cdot.egg-info/top_level.txt +0 -0
  34. {cdot-0.2.30 → cdot-0.2.32}/pyproject.toml +0 -0
  35. {cdot-0.2.30 → cdot-0.2.32}/setup.cfg +0 -0
  36. {cdot-0.2.30 → cdot-0.2.32}/tests/test_json_data_provider_ensembl.py +0 -0
  37. {cdot-0.2.30 → cdot-0.2.32}/tests/test_json_data_provider_refseq.py +0 -0
  38. {cdot-0.2.30 → cdot-0.2.32}/tests/test_pyhgvs.py +0 -0
  39. {cdot-0.2.30 → cdot-0.2.32}/tests/test_rest_data_provider_prefetch.py +0 -0
  40. {cdot-0.2.30 → cdot-0.2.32}/tests/test_struct_dict_compat.py +0 -0
  41. {cdot-0.2.30 → cdot-0.2.32}/tests/test_version_safety.py +0 -0
  42. {cdot-0.2.30 → cdot-0.2.32}/tests/test_version_safety_real_records.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cdot
3
- Version: 0.2.30
3
+ Version: 0.2.32
4
4
  Summary: Transcripts for HGVS libraries
5
5
  Author-email: Dave Lawrence <davmlaw@gmail.com>
6
6
  License: MIT
@@ -47,6 +47,7 @@ Dynamic: license-file
47
47
  [HGVS](http://varnomen.hgvs.org/) variants - the gene/transcript coordinates, exon structure and
48
48
  genome alignments - for the two most popular Python HGVS libraries:
49
49
  [biocommons HGVS](https://github.com/biocommons/hgvs) and [PyHGVS](https://github.com/counsyl/hgvs).
50
+ It also provides tools to repair the malformed or outdated HGVS strings common in real-world data.
50
51
 
51
52
  To do HGVS work (e.g. convert `NM_001637.3:c.1582G>A` to genomic coordinates) those libraries need a
52
53
  transcript data source. The usual source, [UTA](https://github.com/biocommons/uta), is a PostgreSQL
@@ -60,7 +61,18 @@ Because it reads the released annotation files directly, cdot covers **1.58 mill
60
61
  alignments**, including historical transcript versions - vs ~141k in UTA (v.20210129) - which matters
61
62
  when resolving legacy HGVS. See [cdot vs UTA](https://github.com/SACGF/cdot/blob/main/docs/cdot_vs_uta.md) for the trade-offs.
62
63
 
63
- Recent changes are in the [changelog](https://github.com/SACGF/cdot/blob/main/CHANGELOG.md).
64
+ Transcript data alone doesn't get real-world HGVS resolved. Strings pasted from reports, spreadsheets and
65
+ old lab archives are often slightly malformed, name a gene instead of a transcript, or cite a transcript
66
+ version that has since been retired. cdot also includes tools for this in `cdot.hgvs` (see
67
+ [Fixing real-world HGVS](#fixing-real-world-hgvs)):
68
+
69
+ * **Cleaning** - `clean_hgvs()` repairs common formatting errors (whitespace, case, punctuation typos,
70
+ swapped gene/transcript, trailing `p.` annotations) and reports every change it made.
71
+ * **Gene symbols** - `BRCA2:c.68_69del` resolves to the MANE Select transcript.
72
+ * **Version substitution** - opt-in: if the cited transcript version isn't available, use an adjacent
73
+ version, but only when a check confirms the variant's genomic coordinate won't change.
74
+
75
+ Recent changes are in the [changelog](https://github.com/SACGF/cdot/blob/main/CHANGELOG.md), and changes to the published transcript data in the [data changelog](https://github.com/SACGF/cdot/blob/main/CHANGELOG-data.md).
64
76
 
65
77
  ## Install
66
78
 
@@ -104,7 +116,7 @@ am.c_to_g(var_c)
104
116
  > [`FastaSeqFetcher`](https://github.com/SACGF/cdot/blob/main/docs/fasta_seqfetcher.md) (chained after SeqRepo) so every cdot transcript
105
117
  > resolves against a local genome FASTA.
106
118
 
107
- For fixing messy HGVS input and fast bulk processing, see [Advanced usage](https://github.com/SACGF/cdot/blob/main/docs/advanced_usage.md).
119
+ For fast bulk processing over the REST API, see [read-ahead batch retrieval](https://github.com/SACGF/cdot/blob/main/docs/advanced_usage.md#read-ahead-batch-retrieval).
108
120
 
109
121
  [PyHGVS](https://github.com/counsyl/hgvs) example (needs `pip install 'cdot[fasta]'` for pysam):
110
122
 
@@ -121,6 +133,46 @@ pyhgvs.parse_hgvs_name('NM_001637.3:c.1582G>A', genome, get_transcript=factory.g
121
133
 
122
134
  [more PyHGVS examples](https://github.com/SACGF/cdot/blob/main/docs/examples_pyhgvs.md):
123
135
 
136
+ ## Fixing real-world HGVS
137
+
138
+ biocommons HGVS rejects strings that are almost, but not quite, valid. `fix_hgvs()` cleans them up first
139
+ and returns a list of `HGVSFix` records (severity, code, message) so you can decide whether to trust,
140
+ log or reject the result:
141
+
142
+ ```python
143
+ from cdot.hgvs import fix_hgvs
144
+
145
+ result, fixes = fix_hgvs("BRCA2(NM_000059.4):c.68_69DEL p.(Glu23fs)")
146
+ # result = "NM_000059.4(BRCA2):c.68_69del"
147
+ for fix in fixes:
148
+ print(fix)
149
+ # Removed trailing protein (p.) annotation
150
+ # Lowercased mutation type 'DEL'
151
+ # Swapped gene/transcript
152
+ ```
153
+
154
+ Cleaning is a pure string operation, needing no data or parser, so you can run it ahead of any HGVS
155
+ library. Give `fix_hgvs()` a data provider and genome build and it can also resolve gene symbols and
156
+ (if you opt in) substitute a missing transcript version:
157
+
158
+ ```python
159
+ from cdot.hgvs import fix_hgvs, VersionStrategy
160
+
161
+ # Gene symbol -> MANE Select transcript
162
+ result, fixes = fix_hgvs("BRCA2:c.68_69del", hdp, "GRCh38")
163
+ # result = "NM_000059.4:c.68_69del"
164
+
165
+ # Retired transcript version -> adjacent version, only if coordinate-safe
166
+ result, fixes = fix_hgvs("NM_000059.2:c.36del", hdp, "GRCh38",
167
+ version_fallback=VersionStrategy.UP_THEN_DOWN)
168
+ ```
169
+
170
+ Version substitution is off by default. When it is on, a substitution that can't be verified
171
+ coordinate-safe (same coding structure, alignment gaps and relevant UTR length as the cited version) is
172
+ refused by default, returning an ERROR fix and leaving the string unchanged. See [Advanced usage](https://github.com/SACGF/cdot/blob/main/docs/advanced_usage.md)
173
+ for all the options, and [transcript-version safety](https://github.com/SACGF/cdot/blob/main/docs/transcript_version_safety.md)
174
+ for how the safety check works and the evidence behind it.
175
+
124
176
  ## Documentation
125
177
 
126
178
  See [docs/](https://github.com/SACGF/cdot/tree/main/docs) for reference and how-to guides:
@@ -128,6 +180,7 @@ See [docs/](https://github.com/SACGF/cdot/tree/main/docs) for reference and how-
128
180
  * [JSON data format](https://github.com/SACGF/cdot/blob/main/docs/json_data_format.md) - every field in a cdot JSON(.gz) file
129
181
  * [Coordinates & exon alignments](https://github.com/SACGF/cdot/blob/main/docs/coordinates_and_exons.md) - how exon coordinates and gap strings work
130
182
  * [Advanced usage](https://github.com/SACGF/cdot/blob/main/docs/advanced_usage.md) - fixing messy HGVS input, and bulk read-ahead retrieval
183
+ * [Transcript-version safety](https://github.com/SACGF/cdot/blob/main/docs/transcript_version_safety.md) - when substituting a different transcript version is coordinate-safe
131
184
 
132
185
  See the [docs index](https://github.com/SACGF/cdot/blob/main/docs/README.md) for the full list (examples, FastaSeqFetcher, creating data, cdot vs UTA, …).
133
186
 
@@ -7,6 +7,7 @@
7
7
  [HGVS](http://varnomen.hgvs.org/) variants - the gene/transcript coordinates, exon structure and
8
8
  genome alignments - for the two most popular Python HGVS libraries:
9
9
  [biocommons HGVS](https://github.com/biocommons/hgvs) and [PyHGVS](https://github.com/counsyl/hgvs).
10
+ It also provides tools to repair the malformed or outdated HGVS strings common in real-world data.
10
11
 
11
12
  To do HGVS work (e.g. convert `NM_001637.3:c.1582G>A` to genomic coordinates) those libraries need a
12
13
  transcript data source. The usual source, [UTA](https://github.com/biocommons/uta), is a PostgreSQL
@@ -20,7 +21,18 @@ Because it reads the released annotation files directly, cdot covers **1.58 mill
20
21
  alignments**, including historical transcript versions - vs ~141k in UTA (v.20210129) - which matters
21
22
  when resolving legacy HGVS. See [cdot vs UTA](https://github.com/SACGF/cdot/blob/main/docs/cdot_vs_uta.md) for the trade-offs.
22
23
 
23
- Recent changes are in the [changelog](https://github.com/SACGF/cdot/blob/main/CHANGELOG.md).
24
+ Transcript data alone doesn't get real-world HGVS resolved. Strings pasted from reports, spreadsheets and
25
+ old lab archives are often slightly malformed, name a gene instead of a transcript, or cite a transcript
26
+ version that has since been retired. cdot also includes tools for this in `cdot.hgvs` (see
27
+ [Fixing real-world HGVS](#fixing-real-world-hgvs)):
28
+
29
+ * **Cleaning** - `clean_hgvs()` repairs common formatting errors (whitespace, case, punctuation typos,
30
+ swapped gene/transcript, trailing `p.` annotations) and reports every change it made.
31
+ * **Gene symbols** - `BRCA2:c.68_69del` resolves to the MANE Select transcript.
32
+ * **Version substitution** - opt-in: if the cited transcript version isn't available, use an adjacent
33
+ version, but only when a check confirms the variant's genomic coordinate won't change.
34
+
35
+ Recent changes are in the [changelog](https://github.com/SACGF/cdot/blob/main/CHANGELOG.md), and changes to the published transcript data in the [data changelog](https://github.com/SACGF/cdot/blob/main/CHANGELOG-data.md).
24
36
 
25
37
  ## Install
26
38
 
@@ -64,7 +76,7 @@ am.c_to_g(var_c)
64
76
  > [`FastaSeqFetcher`](https://github.com/SACGF/cdot/blob/main/docs/fasta_seqfetcher.md) (chained after SeqRepo) so every cdot transcript
65
77
  > resolves against a local genome FASTA.
66
78
 
67
- For fixing messy HGVS input and fast bulk processing, see [Advanced usage](https://github.com/SACGF/cdot/blob/main/docs/advanced_usage.md).
79
+ For fast bulk processing over the REST API, see [read-ahead batch retrieval](https://github.com/SACGF/cdot/blob/main/docs/advanced_usage.md#read-ahead-batch-retrieval).
68
80
 
69
81
  [PyHGVS](https://github.com/counsyl/hgvs) example (needs `pip install 'cdot[fasta]'` for pysam):
70
82
 
@@ -81,6 +93,46 @@ pyhgvs.parse_hgvs_name('NM_001637.3:c.1582G>A', genome, get_transcript=factory.g
81
93
 
82
94
  [more PyHGVS examples](https://github.com/SACGF/cdot/blob/main/docs/examples_pyhgvs.md):
83
95
 
96
+ ## Fixing real-world HGVS
97
+
98
+ biocommons HGVS rejects strings that are almost, but not quite, valid. `fix_hgvs()` cleans them up first
99
+ and returns a list of `HGVSFix` records (severity, code, message) so you can decide whether to trust,
100
+ log or reject the result:
101
+
102
+ ```python
103
+ from cdot.hgvs import fix_hgvs
104
+
105
+ result, fixes = fix_hgvs("BRCA2(NM_000059.4):c.68_69DEL p.(Glu23fs)")
106
+ # result = "NM_000059.4(BRCA2):c.68_69del"
107
+ for fix in fixes:
108
+ print(fix)
109
+ # Removed trailing protein (p.) annotation
110
+ # Lowercased mutation type 'DEL'
111
+ # Swapped gene/transcript
112
+ ```
113
+
114
+ Cleaning is a pure string operation, needing no data or parser, so you can run it ahead of any HGVS
115
+ library. Give `fix_hgvs()` a data provider and genome build and it can also resolve gene symbols and
116
+ (if you opt in) substitute a missing transcript version:
117
+
118
+ ```python
119
+ from cdot.hgvs import fix_hgvs, VersionStrategy
120
+
121
+ # Gene symbol -> MANE Select transcript
122
+ result, fixes = fix_hgvs("BRCA2:c.68_69del", hdp, "GRCh38")
123
+ # result = "NM_000059.4:c.68_69del"
124
+
125
+ # Retired transcript version -> adjacent version, only if coordinate-safe
126
+ result, fixes = fix_hgvs("NM_000059.2:c.36del", hdp, "GRCh38",
127
+ version_fallback=VersionStrategy.UP_THEN_DOWN)
128
+ ```
129
+
130
+ Version substitution is off by default. When it is on, a substitution that can't be verified
131
+ coordinate-safe (same coding structure, alignment gaps and relevant UTR length as the cited version) is
132
+ refused by default, returning an ERROR fix and leaving the string unchanged. See [Advanced usage](https://github.com/SACGF/cdot/blob/main/docs/advanced_usage.md)
133
+ for all the options, and [transcript-version safety](https://github.com/SACGF/cdot/blob/main/docs/transcript_version_safety.md)
134
+ for how the safety check works and the evidence behind it.
135
+
84
136
  ## Documentation
85
137
 
86
138
  See [docs/](https://github.com/SACGF/cdot/tree/main/docs) for reference and how-to guides:
@@ -88,6 +140,7 @@ See [docs/](https://github.com/SACGF/cdot/tree/main/docs) for reference and how-
88
140
  * [JSON data format](https://github.com/SACGF/cdot/blob/main/docs/json_data_format.md) - every field in a cdot JSON(.gz) file
89
141
  * [Coordinates & exon alignments](https://github.com/SACGF/cdot/blob/main/docs/coordinates_and_exons.md) - how exon coordinates and gap strings work
90
142
  * [Advanced usage](https://github.com/SACGF/cdot/blob/main/docs/advanced_usage.md) - fixing messy HGVS input, and bulk read-ahead retrieval
143
+ * [Transcript-version safety](https://github.com/SACGF/cdot/blob/main/docs/transcript_version_safety.md) - when substituting a different transcript version is coordinate-safe
91
144
 
92
145
  See the [docs index](https://github.com/SACGF/cdot/blob/main/docs/README.md) for the full list (examples, FastaSeqFetcher, creating data, cdot vs UTA, …).
93
146
 
@@ -1,4 +1,4 @@
1
- __version__ = "0.2.30"
1
+ __version__ = "0.2.32"
2
2
  # Data version is kept in generate_transcript_version.json_schema_version
3
3
 
4
4
  def get_data_schema_int(version: str) -> int:
@@ -22,7 +22,9 @@ from cdot.hgvs.gene_hgvs import (
22
22
  fix_hgvs,
23
23
  rank_transcripts_for_gene,
24
24
  resolve_gene_hgvs,
25
+ resolve_missing_accession_prefix,
25
26
  resolve_transcript_version,
26
27
  UnsafeVersionPolicy,
27
28
  )
28
29
  from cdot.hgvs.version_safety import intrinsic_cds_structure
30
+ from cdot.hgvs.translation import fix_c_to_p
@@ -54,6 +54,7 @@ class HGVSFixCode(Enum):
54
54
  DROPPED_GENOMIC_REF_IN_PARENS = "dropped_genomic_ref_in_parens"
55
55
  ADDED_TRANSCRIPT_UNDERSCORE = "added_transcript_underscore"
56
56
  FIXED_MULTIPLE_UNDERSCORE = "fixed_multiple_underscore"
57
+ DROPPED_EMPTY_VERSION = "dropped_empty_version"
57
58
  UPPERCASED_BASES = "uppercased_bases"
58
59
  RECONSTRUCTED_STRUCTURE = "reconstructed_structure"
59
60
  ADDED_MISSING_KIND = "added_missing_kind"
@@ -74,6 +75,18 @@ class HGVSFixCode(Enum):
74
75
  # Gene → transcript resolution (WARNING on success, ERROR on failure)
75
76
  RESOLVED_GENE_TO_TRANSCRIPT = "resolved_gene_to_transcript"
76
77
  NO_TRANSCRIPT_FOR_GENE = "no_transcript_for_gene"
78
+ # Accession prefix restoration for a bare-number accession, verified against
79
+ # a data provider (see gene_hgvs.resolve_missing_accession_prefix; WARNING)
80
+ ADDED_ACCESSION_PREFIX = "added_accession_prefix"
81
+ # c. to p. translation checks from cdot's translation data (see translation.fix_c_to_p)
82
+ USED_TRANSLATION_TABLE = "used_translation_table" # WARNING
83
+ NEW_UGA_READ_AS_STOP = "new_uga_read_as_stop" # WARNING
84
+ UGA_MAY_BE_READ_AS_SEC = "uga_may_be_read_as_sec" # WARNING
85
+ TRANSLATION_EXCEPTION_NOT_APPLIED = "translation_exception_not_applied" # WARNING
86
+ TRANSLATION_DATA_INCOMPLETE = "translation_data_incomplete" # WARNING
87
+ REFERENCE_CODON_MISMATCH = "reference_codon_mismatch" # WARNING, ERROR if it makes a stop
88
+ UNSUPPORTED_TRANSLATION_TABLE = "unsupported_translation_table" # ERROR
89
+ RIBOSOMAL_SLIPPAGE_UNSUPPORTED = "ribosomal_slippage_unsupported" # ERROR
77
90
  # Validation errors (ERROR)
78
91
  NO_COLON = "no_colon"
79
92
  MISSING_REFERENCE_SEQUENCE = "missing_reference_sequence"
@@ -92,6 +105,7 @@ class HGVSCleanOp(Enum):
92
105
  """
93
106
  STRIP_PROTEIN_SUFFIX = "strip_protein_suffix"
94
107
  STRIP_LEADING_JUNK = "strip_leading_junk"
108
+ FIX_SPACE_BEFORE_KIND = "fix_space_before_kind"
95
109
  STRIP_WHITESPACE = "strip_whitespace"
96
110
  STRIP_SURROUNDING_PUNCTUATION = "strip_surrounding_punctuation"
97
111
  FIX_MULTIPLE_COLON = "fix_multiple_colon"
@@ -101,6 +115,7 @@ class HGVSCleanOp(Enum):
101
115
  FIX_PREFIX_COLON = "fix_prefix_colon"
102
116
  FIX_MULTIPLE_KIND = "fix_multiple_kind"
103
117
  FIX_SEPARATOR_TYPO = "fix_separator_typo"
118
+ DROP_EMPTY_VERSION = "drop_empty_version"
104
119
  ADD_N_PREFIX = "add_n_prefix"
105
120
  LOWERCASE_MUTATION_TYPE = "lowercase_mutation_type"
106
121
  DROP_DEL_DUP_COUNT = "drop_del_dup_count"
@@ -244,9 +259,10 @@ _PREFIX_COLON_UNDERSCORE = re.compile(r"^(N[MR]|X[MR]):_", re.IGNORECASE)
244
259
  # Transcript accession prefixes (used to locate the HGVS core within a string)
245
260
  _TX_PREFIX = r"(?:NM_|NR_|XM_|XR_|NC_|NG_|ENST|LRG_)"
246
261
 
247
- # Leading junk before a transcript accession, e.g. "#", ":", a literal "\t",
248
- # or a genome-build prefix like "GRCh38.p2 " / "GRCh37.p12#"
249
- _LEADING_JUNK = re.compile(r"^(?:GRCh\d+(?:\.p\d+)?[#\s]*|\\t|[#:])+", re.IGNORECASE)
262
+ # Leading junk before a transcript accession, e.g. "#", ":", "-", a literal
263
+ # "\t", whitespace (": NM_..."), or a genome-build prefix like "GRCh38.p2 " /
264
+ # "GRCh37.p12#". Only stripped when a transcript accession actually follows.
265
+ _LEADING_JUNK = re.compile(r"^(?:GRCh\d+(?:\.p\d+)?[#\s]*|\\t|[#:\-\s])+", re.IGNORECASE)
250
266
 
251
267
  # Gene symbol wedged between extra colons, e.g. "tx:(GENE):c." / "tx:(GENE)c." /
252
268
  # "tx:GENE:c." — should be "tx(GENE):c."
@@ -270,6 +286,26 @@ _KIND_COLON = re.compile(r":([cgnmp]):(?=[\d*?(\-])", re.IGNORECASE)
270
286
  # Period used in place of the substitution '>', e.g. "1030C.T" -> "1030C>T"
271
287
  _SUB_PERIOD = re.compile(r"([ACGT])\.([ACGT])$", re.IGNORECASE)
272
288
 
289
+ # Semicolon or underscore used in place of the accession:allele colon, e.g.
290
+ # "NM_000059.4;c.68del" or "NM_000059.4_c.68del". Anchored to the start so a
291
+ # ';' inside a multi-variant allele is never touched. LRG is excluded (its
292
+ # accessions legitimately contain underscores).
293
+ _ACCESSION_KIND_SEPARATOR = re.compile(
294
+ r"^((?:NM_|NR_|XM_|XR_|NC_|NG_|ENST)\d+(?:\.\d+)?)[;_](?=[cgnmp]\.)", re.IGNORECASE)
295
+
296
+ # Whitespace used in place of the reference:kind colon after a bare gene symbol,
297
+ # e.g. "BRCA2 c.68del" -> "BRCA2:c.68del". Restricted to a plain symbol (no
298
+ # '_'/'.'/':') so accessions are left to the whitespace/reconstruction steps,
299
+ # and runs before whitespace stripping (which would otherwise glue the symbol to
300
+ # the kind letter, mangling symbols that resemble an accession, eg NR2E3).
301
+ _SPACE_BEFORE_KIND = re.compile(r"^([A-Za-z][A-Za-z0-9]{1,9})\s+(?=[cgnmp]\.)", re.IGNORECASE)
302
+
303
+ # An empty version field: accession, a dot, then no digits before the colon,
304
+ # e.g. "NM_001754.:c.749G>A". The version digits were dropped, so drop the dot
305
+ # and leave a valid unversioned accession.
306
+ _EMPTY_VERSION = re.compile(
307
+ r"^((?:NM_|NR_|XM_|XR_|NC_|NG_|ENST)\d+)\.(?=:)", re.IGNORECASE)
308
+
273
309
  # del/dup with an explicit range followed by a redundant base count, e.g.
274
310
  # "c.1315_1337dup23" -> "c.1315_1337dup" (the range already gives the length;
275
311
  # normalise to plain del/dup). The range guard avoids changing the meaning of a
@@ -723,12 +759,45 @@ def _op_fix_gene_wrapper(s: str) -> tuple[str, list[HGVSFix]]:
723
759
  return s, []
724
760
 
725
761
 
762
+ def _op_fix_space_before_kind(s: str) -> tuple[str, list[HGVSFix]]:
763
+ # Whitespace in place of the reference:kind colon after a bare gene symbol,
764
+ # e.g. "BRCA2 c.68del" -> "BRCA2:c.68del". Must run before whitespace
765
+ # stripping, which would glue the symbol to the kind letter and leave
766
+ # accession-lookalike symbols (eg NR2E3) unrepairable.
767
+ s2 = _SPACE_BEFORE_KIND.sub(r"\1:", s)
768
+ if s2 != s:
769
+ return s2, [HGVSFix(
770
+ severity=HGVSFixSeverity.WARNING,
771
+ code=HGVSFixCode.FIXED_SEPARATOR_TYPO,
772
+ message="Replaced space between reference and kind with ':'",
773
+ original=s, fixed=s2,
774
+ )]
775
+ return s, []
776
+
777
+
778
+ def _op_drop_empty_version(s: str) -> tuple[str, list[HGVSFix]]:
779
+ # An accession with a dot but no version digits, e.g. "NM_001754.:c.749G>A"
780
+ # -> "NM_001754:c.749G>A". The version was dropped; an unversioned accession
781
+ # is valid HGVS, so drop the dangling dot rather than invent a version.
782
+ s2 = _EMPTY_VERSION.sub(r"\1", s)
783
+ if s2 != s:
784
+ return s2, [HGVSFix(
785
+ severity=HGVSFixSeverity.WARNING,
786
+ code=HGVSFixCode.DROPPED_EMPTY_VERSION,
787
+ message="Dropped dangling dot of an empty transcript version",
788
+ original=s, fixed=s2,
789
+ )]
790
+ return s, []
791
+
792
+
726
793
  def _op_fix_separator_typo(s: str) -> tuple[str, list[HGVSFix]]:
727
- # Comma ("c,1811") or colon ("c:1811") in place of the kind dot, and period
728
- # in place of the substitution '>' ("1030C.T" -> "1030C>T").
794
+ # Comma ("c,1811") or colon ("c:1811") in place of the kind dot, period
795
+ # in place of the substitution '>' ("1030C.T" -> "1030C>T"), and semicolon
796
+ # or underscore in place of the accession:allele colon ("NM_000059.4;c.68del").
729
797
  s2 = _KIND_COMMA.sub(r":\1.", s)
730
798
  s2 = _KIND_COLON.sub(r":\1.", s2)
731
799
  s2 = _SUB_PERIOD.sub(r"\1>\2", s2)
800
+ s2 = _ACCESSION_KIND_SEPARATOR.sub(r"\1:", s2)
732
801
  if s2 != s:
733
802
  return s2, [HGVSFix(
734
803
  severity=HGVSFixSeverity.WARNING,
@@ -777,6 +846,7 @@ def _op_drop_genomic_ref_in_parens(s: str) -> tuple[str, list[HGVSFix]]:
777
846
  _PIPELINE: list[tuple[HGVSCleanOp, "callable"]] = [
778
847
  (HGVSCleanOp.STRIP_PROTEIN_SUFFIX, _op_strip_protein_suffix),
779
848
  (HGVSCleanOp.STRIP_LEADING_JUNK, _op_strip_leading_junk),
849
+ (HGVSCleanOp.FIX_SPACE_BEFORE_KIND, _op_fix_space_before_kind),
780
850
  (HGVSCleanOp.STRIP_WHITESPACE, _op_strip_whitespace),
781
851
  (HGVSCleanOp.STRIP_SURROUNDING_PUNCTUATION, _op_strip_surrounding_punctuation),
782
852
  (HGVSCleanOp.FIX_MULTIPLE_COLON, _op_fix_multiple_colon),
@@ -786,6 +856,7 @@ _PIPELINE: list[tuple[HGVSCleanOp, "callable"]] = [
786
856
  (HGVSCleanOp.FIX_PREFIX_COLON, _op_fix_prefix_colon),
787
857
  (HGVSCleanOp.FIX_MULTIPLE_KIND, _op_fix_multiple_kind),
788
858
  (HGVSCleanOp.FIX_SEPARATOR_TYPO, _op_fix_separator_typo),
859
+ (HGVSCleanOp.DROP_EMPTY_VERSION, _op_drop_empty_version),
789
860
  (HGVSCleanOp.ADD_N_PREFIX, _op_add_n_prefix),
790
861
  (HGVSCleanOp.LOWERCASE_MUTATION_TYPE, _op_lowercase_mutation_type),
791
862
  (HGVSCleanOp.DROP_DEL_DUP_COUNT, _op_drop_del_dup_count),
@@ -2,7 +2,7 @@ import abc
2
2
  import gzip
3
3
  import requests
4
4
 
5
- from collections import defaultdict
5
+ from collections import defaultdict, OrderedDict
6
6
  from lazy import lazy
7
7
  from hgvs.dataproviders.interface import Interface
8
8
  from hgvs.dataproviders.seqfetcher import SeqFetcher
@@ -31,6 +31,12 @@ class AbstractJSONDataProvider(Interface):
31
31
  # All cdot data is 'splign', it's the method used in NCBI/Ensembl GTFs, and we also only pull out 'splign' from UTA
32
32
  NCBI_ALN_METHOD = "splign"
33
33
  required_version = "1.1"
34
+ # get_tx_exons() converts cdot's stored exon arrays into the list of per-exon dicts
35
+ # (with CIGARs) that biocommons hgvs expects. That conversion is the per-variant hot
36
+ # path, and its input is immutable once loaded/fetched, so each provider instance
37
+ # keeps a small LRU of built results (see get_tx_exons). Size is a per-instance
38
+ # bound on distinct (tx_ac, alt_ac) entries; each entry is just the exon dict list.
39
+ TX_EXONS_CACHE_SIZE = 10_000
34
40
 
35
41
  def __init__(self, assemblies: List[str] = None, mode=None, cache=None, seqfetcher=None):
36
42
  """ assemblies: defaults to ["GRCh37", "GRCh38"]
@@ -54,12 +60,32 @@ class AbstractJSONDataProvider(Interface):
54
60
  self.assembly_by_contig = {}
55
61
  for assembly_name, contig_map in self.assembly_maps.items():
56
62
  self.assembly_by_contig.update({contig: assembly_name for contig in contig_map.keys()})
63
+ # Per-instance LRU for get_tx_exons results, keyed (tx_ac, alt_ac). Instance-level
64
+ # (not functools.lru_cache on the method) so the cache dies with the instance and
65
+ # is never shared across providers holding different data.
66
+ self._tx_exons_cache = OrderedDict()
57
67
 
58
68
 
59
69
  @abc.abstractmethod
60
70
  def _get_transcript(self, tx_ac):
61
71
  pass
62
72
 
73
+ def _get_transcripts(self, tx_acs) -> dict:
74
+ """
75
+ Return ``{tx_ac: transcript_data}`` for the given accessions.
76
+
77
+ This is the retrieve-many hook: every method that walks a list of transcripts
78
+ (:meth:`get_tx_for_gene`, :meth:`get_tx_ac_tags_for_gene`,
79
+ :meth:`get_tx_for_region`, :meth:`_get_tags_by_tx_ac`) goes through here, so a
80
+ provider that can answer for many accessions at once - a single database query,
81
+ a bulk REST call - only has to override this one method to speed all of them up.
82
+
83
+ The default fetches one at a time via :meth:`_get_transcript`, which is what a
84
+ provider holding its transcripts in memory already wants. Duplicate accessions
85
+ collapse (it is keyed by accession) and insertion order follows ``tx_acs``.
86
+ """
87
+ return {tx_ac: self._get_transcript(tx_ac) for tx_ac in tx_acs}
88
+
63
89
  @abc.abstractmethod
64
90
  def _get_gene(self, gene):
65
91
  pass
@@ -283,7 +309,17 @@ class AbstractJSONDataProvider(Interface):
283
309
  return "".join(cigar_ops)
284
310
 
285
311
  def get_tx_exons(self, tx_ac, alt_ac, alt_aln_method):
286
- self._check_alt_aln_method(alt_aln_method)
312
+ self._check_alt_aln_method(alt_aln_method) # raises unless alt_aln_method == splign
313
+ # alt_aln_method is validated above (always splign) so it is not part of the key.
314
+ # Only successful builds are cached: a None result can become available later on
315
+ # providers that fetch lazily (eg RESTDataProvider after prefetch()).
316
+ # Callers get the cached list itself and must not mutate it (biocommons hgvs
317
+ # only reads it; sorted() takes a copy).
318
+ cache_key = (tx_ac, alt_ac)
319
+ if (cached := self._tx_exons_cache.get(cache_key)) is not None:
320
+ self._tx_exons_cache.move_to_end(cache_key)
321
+ return cached
322
+
287
323
  transcript = self._get_transcript(tx_ac)
288
324
  if not transcript:
289
325
  return None
@@ -320,6 +356,9 @@ class AbstractJSONDataProvider(Interface):
320
356
  }
321
357
  tx_exons.append(exon_data)
322
358
 
359
+ self._tx_exons_cache[cache_key] = tx_exons
360
+ if len(self._tx_exons_cache) > self.TX_EXONS_CACHE_SIZE:
361
+ self._tx_exons_cache.popitem(last=False)
323
362
  return tx_exons
324
363
 
325
364
  def get_tx_identity_info(self, tx_ac):
@@ -457,8 +496,7 @@ class LocalDataProvider(AbstractJSONDataProvider):
457
496
  """ return transcript info records for supplied gene, in order of decreasing length """
458
497
 
459
498
  tx_list = [] # Store in tuples with length, so we can sort before returning
460
- for transcript_id in self._get_transcript_ids_for_gene(gene):
461
- transcript_data = self._get_transcript(transcript_id)
499
+ for transcript_id, transcript_data in self._get_transcripts(self._get_transcript_ids_for_gene(gene)).items():
462
500
  cds_start_i = transcript_data.get("start_codon")
463
501
  cds_end_i = transcript_data.get("stop_codon")
464
502
  # Return the canonical versioned accession from the record (matching
@@ -494,8 +532,9 @@ class LocalDataProvider(AbstractJSONDataProvider):
494
532
  stores MANE/canonical status separately from the transcript JSON). The
495
533
  override receives the accession, the full transcript dict, and the build
496
534
  name, so it can fall back to the cdot data when the external source has no
497
- entry. (To answer for many accessions in one query, override
498
- :meth:`_get_tags_by_tx_ac` instead.)
535
+ entry. (To batch the transcript retrieval that feeds this, override
536
+ :meth:`_get_transcripts`; to answer the tags themselves for many accessions
537
+ in one query, override :meth:`_get_tags_by_tx_ac`.)
499
538
 
500
539
  Example override (Django)::
501
540
 
@@ -514,16 +553,17 @@ class LocalDataProvider(AbstractJSONDataProvider):
514
553
  """
515
554
  Return {tx_ac: tags} for the given accessions in the given genome build.
516
555
 
517
- The default loops :meth:`_get_transcript` + :meth:`_get_transcript_tags`
518
- per accession. Override in a subclass that can answer for all accessions
519
- in a single query (e.g. a Django provider with a MANE table) to avoid the
520
- N+1 query pattern on the gene-symbol resolution path.
556
+ The default retrieves the transcripts in one go with :meth:`_get_transcripts`
557
+ and reads each one's tags with :meth:`_get_transcript_tags`, so a provider
558
+ that overrides :meth:`_get_transcripts` already answers this in a single
559
+ query - there is no need to override this method just to avoid the N+1.
560
+
561
+ Override this one when the tags live somewhere other than the transcript
562
+ JSON and can be answered for all accessions at once (e.g. a Django provider
563
+ with a MANE table), which saves retrieving the transcripts at all.
521
564
  """
522
- result = {}
523
- for tx_ac in tx_acs:
524
- transcript_data = self._get_transcript(tx_ac)
525
- result[tx_ac] = self._get_transcript_tags(tx_ac, transcript_data, genome_build)
526
- return result
565
+ return {tx_ac: self._get_transcript_tags(tx_ac, transcript_data, genome_build)
566
+ for tx_ac, transcript_data in self._get_transcripts(tx_acs).items()}
527
567
 
528
568
  def get_tx_ac_tags_for_gene(self, gene: str, genome_build: str) -> list[tuple[str, list[str]]]:
529
569
  """
@@ -546,8 +586,7 @@ class LocalDataProvider(AbstractJSONDataProvider):
546
586
  are unaffected, as ``transcript_data["id"]`` equals that id.
547
587
  """
548
588
  lengths = []
549
- for transcript_id in self._get_transcript_ids_for_gene(gene):
550
- transcript_data = self._get_transcript(transcript_id)
589
+ for transcript_id, transcript_data in self._get_transcripts(self._get_transcript_ids_for_gene(gene)).items():
551
590
  build_data = transcript_data["genome_builds"].get(genome_build)
552
591
  if build_data is None:
553
592
  continue
@@ -571,9 +610,8 @@ class LocalDataProvider(AbstractJSONDataProvider):
571
610
 
572
611
  tx_list = []
573
612
  contig_iv_tree = self._get_contig_interval_tree(alt_ac)
574
- for interval in contig_iv_tree[start_i:end_i+1]:
575
- transcript_id = interval.data
576
- transcript_data = self._get_transcript(transcript_id)
613
+ transcript_ids = [interval.data for interval in contig_iv_tree[start_i:end_i+1]]
614
+ for transcript_id, transcript_data in self._get_transcripts(transcript_ids).items():
577
615
  build_data = self._get_transcript_coordinates_for_contig(transcript_data, alt_ac)
578
616
  contig, tx_start, tx_end, strand = self._get_contig_start_end_strand(build_data)
579
617
  if contig == alt_ac:
@@ -11,6 +11,7 @@ resolve_gene_hgvs() in one call — the recommended function for most callers.
11
11
  """
12
12
 
13
13
  import inspect
14
+ import re
14
15
  from enum import Enum
15
16
  from typing import Optional
16
17
 
@@ -343,6 +344,105 @@ def resolve_gene_hgvs(
343
344
  return resolved, fixes
344
345
 
345
346
 
347
+ # ---------------------------------------------------------------------------
348
+ # Public: resolve_missing_accession_prefix — restore a dropped RefSeq prefix
349
+ # ---------------------------------------------------------------------------
350
+
351
+ # A bare-number accession: the user dropped the RefSeq prefix entirely (or left
352
+ # only its underscore), e.g. "000059.4:c.68del" / "_001128425.1:c.667A>G".
353
+ # RefSeq digit fields are 6 or 9 digits, but 7-8 are accepted as a 9-digit
354
+ # accession missing its leading zeros (see _digit_field_candidates).
355
+ _BARE_NUMBER_ACCESSION = re.compile(r"^_?(\d{6,9})((?:\.\d+)?):([cn])\.", re.IGNORECASE)
356
+
357
+ # The kind letter constrains which RefSeq prefixes could have been dropped:
358
+ # a coding "c." implies an mRNA accession, a non-coding "n." an RNA accession.
359
+ _PREFIXES_FOR_KIND = {
360
+ "c": ("NM_", "XM_"),
361
+ "n": ("NR_", "XR_"),
362
+ }
363
+
364
+
365
+ def _digit_field_candidates(digits: str) -> list[str]:
366
+ """Candidate digit fields for a bare-number accession.
367
+
368
+ RefSeq accessions have 6 or 9 digits after the prefix. A 6 or 9 digit
369
+ input is taken as written; a 7-8 digit input can only be a 9-digit
370
+ accession whose leading zeros were also dropped, so it is zero-padded.
371
+ """
372
+ if len(digits) in (6, 9):
373
+ return [digits]
374
+ return [digits.zfill(9)]
375
+
376
+
377
+ def resolve_missing_accession_prefix(
378
+ hgvs_string: str,
379
+ data_provider,
380
+ genome_build: Optional[str] = None,
381
+ ) -> tuple[str, list[HGVSFix]]:
382
+ """
383
+ Restore a RefSeq prefix that was dropped from the transcript accession,
384
+ e.g. "000059.4:c.68del" → "NM_000059.4:c.68del".
385
+
386
+ The kind letter narrows the possibilities ("c." implies NM_/XM_, "n."
387
+ implies NR_/XR_) but cannot fully disambiguate, so the candidates are
388
+ checked against the data provider (via ``get_tx_versions``) and the fix is
389
+ applied only when exactly one candidate accession exists there. With zero
390
+ or several matches the string is returned unchanged - this function never
391
+ guesses.
392
+
393
+ Returns:
394
+ (resolved_string, fixes)
395
+ fixes contains a WARNING ADDED_ACCESSION_PREFIX HGVSFix when the prefix
396
+ was restored, and is empty otherwise (no bare-number accession, no
397
+ provider match, ambiguous match, or a provider that cannot enumerate
398
+ versions).
399
+
400
+ ``genome_build`` is optional and only used to restrict the existence check
401
+ to accessions placeable in that build (for providers whose
402
+ ``get_tx_versions`` accepts it).
403
+
404
+ Example::
405
+
406
+ resolved, fixes = resolve_missing_accession_prefix(
407
+ "000059.4:c.68del", data_provider)
408
+ # resolved = "NM_000059.4:c.68del" (if only NM_000059 exists)
409
+ # fixes[0].code = HGVSFixCode.ADDED_ACCESSION_PREFIX
410
+ """
411
+ m = _BARE_NUMBER_ACCESSION.match(hgvs_string)
412
+ if m is None:
413
+ return hgvs_string, []
414
+
415
+ digits, version, kind = m.group(1), m.group(2), m.group(3).lower()
416
+ candidates = []
417
+ for prefix in _PREFIXES_FOR_KIND[kind]:
418
+ for digit_field in _digit_field_candidates(digits):
419
+ accession = f"{prefix}{digit_field}"
420
+ try:
421
+ versions = _get_tx_versions(data_provider, accession, genome_build)
422
+ except NotImplementedError:
423
+ return hgvs_string, [] # provider can't enumerate, don't guess
424
+ if versions:
425
+ candidates.append(accession)
426
+
427
+ if len(candidates) != 1:
428
+ return hgvs_string, [] # unknown or ambiguous, never guess
429
+
430
+ accession = candidates[0]
431
+ original_head = hgvs_string[:m.end(2)]
432
+ fixed_head = f"{accession}{version}"
433
+ resolved = fixed_head + hgvs_string[m.end(2):]
434
+ return resolved, [HGVSFix(
435
+ severity=HGVSFixSeverity.WARNING,
436
+ code=HGVSFixCode.ADDED_ACCESSION_PREFIX,
437
+ message=(
438
+ f"Restored missing accession prefix: '{original_head}' → "
439
+ f"'{fixed_head}' (unique match in data provider)"
440
+ ),
441
+ original=original_head,
442
+ fixed=fixed_head,
443
+ )]
444
+
445
+
346
446
  # ---------------------------------------------------------------------------
347
447
  # Public: resolve_transcript_version — adjacent-version fallback
348
448
  # ---------------------------------------------------------------------------
@@ -540,9 +640,10 @@ def fix_hgvs(
540
640
  """
541
641
  Clean and resolve an HGVS string in one call.
542
642
 
543
- Runs clean_hgvs() first (always), then resolve_gene_hgvs() if a
544
- data_provider and genome_build are supplied. All fixes from both steps
545
- are returned in a single list.
643
+ Runs clean_hgvs() first (always), then resolve_missing_accession_prefix()
644
+ if a data_provider is supplied, then resolve_gene_hgvs() if a data_provider
645
+ and genome_build are supplied. All fixes from every step are returned in a
646
+ single list.
546
647
 
547
648
  If data_provider/genome_build are omitted, only string cleaning is performed.
548
649
  This is useful when the caller knows the input already contains a transcript.
@@ -586,6 +687,14 @@ def fix_hgvs(
586
687
  """
587
688
  result, fixes = clean_hgvs(hgvs_string, ops=ops)
588
689
 
690
+ if data_provider is not None:
691
+ # Restore a dropped RefSeq prefix (eg "000059.4:c.68del") before gene
692
+ # resolution, so the bare number is not mistaken for a gene symbol.
693
+ result, prefix_fixes = resolve_missing_accession_prefix(
694
+ result, data_provider, genome_build=genome_build,
695
+ )
696
+ fixes.extend(prefix_fixes)
697
+
589
698
  if data_provider is not None and genome_build is not None:
590
699
  result, resolution_fixes = resolve_gene_hgvs(
591
700
  result, data_provider, genome_build,