pysec2pri 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/PKG-INFO +34 -12
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/README.md +32 -10
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/pyproject.toml +3 -3
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/api.py +2 -2
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/cli.py +12 -3
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/ensembl.yaml +0 -7
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/ncbi.yaml +0 -8
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/vgnc.yaml +0 -16
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/consolidate.py +86 -18
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/hmdb.py +21 -36
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/version.py +1 -1
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/LICENSE +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/__init__.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/__main__.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/__init__.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/chebi.yaml +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/hgnc.yaml +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/hmdb_metabolites.yaml +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/hmdb_proteins.yaml +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/uniprot.yaml +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/config/wikidata.yaml +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/constants.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/download.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/downloads/__init__.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/downloads/chebi.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/downloads/ensembl.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/downloads/hgnc.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/downloads/hmdb.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/downloads/ncbi.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/downloads/uniprot.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/exports.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/logging.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/__init__.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/base.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/chebi.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/ensembl.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/hgnc.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/ncbi.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/uniprot.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/vgnc.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/parsers/wikidata.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/py.typed +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/queries/__init__.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/queries/chemical_redirects.rq +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/queries/chemical_redirects_test.rq +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/queries/gene_redirects.rq +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/queries/gene_redirects_test.rq +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/queries/protein_redirects.rq +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/queries/protein_redirects_test.rq +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/update_ids.py +0 -0
- {pysec2pri-0.2.0 → pysec2pri-0.2.2}/src/pysec2pri/xref.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pysec2pri
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Secondary to primary identifier mapping
|
|
5
5
|
Keywords: snekpack,cookiecutter
|
|
6
6
|
Author: Javier Millán Acosta
|
|
@@ -40,7 +40,7 @@ Requires-Dist: numpy
|
|
|
40
40
|
Requires-Dist: rdkit
|
|
41
41
|
Requires-Dist: polars-runtime-32
|
|
42
42
|
Requires-Dist: sssom-schema==1.1.0a5
|
|
43
|
-
Requires-Dist: mapkgsutils>=0.0.
|
|
43
|
+
Requires-Dist: mapkgsutils>=0.0.5
|
|
44
44
|
Maintainer: Javier Millán Acosta
|
|
45
45
|
Maintainer-email: Javier Millán Acosta <javier.millan.acosta@gmail.com>
|
|
46
46
|
Requires-Python: >=3.10
|
|
@@ -96,20 +96,33 @@ uv pip install git+https://github.com/jmillanacosta/pysec2pri.git
|
|
|
96
96
|
|
|
97
97
|
### Generating mapping sets
|
|
98
98
|
|
|
99
|
-
|
|
99
|
+
Mapping sets can be generated from bash:
|
|
100
100
|
|
|
101
101
|
```bash
|
|
102
|
-
|
|
102
|
+
# ID mapping sets
|
|
103
|
+
pysec2pri chebi ids (--help)
|
|
104
|
+
pysec2pri ensembl ids (--help)
|
|
105
|
+
pysec2pri hgnc ids (--help)
|
|
106
|
+
pysec2pri vgnc ids (--help)
|
|
107
|
+
pysec2pri ncbi ids (--help)
|
|
108
|
+
pysec2pri hmdb-protein ids (--help)
|
|
109
|
+
pysec2pri hmdb-gene ids (--help)
|
|
110
|
+
pysec2pri uniprot ids (--help)
|
|
111
|
+
# Label mapping sets
|
|
112
|
+
pysec2pri chebi labels (--help)
|
|
113
|
+
pysec2pri hgnc labels (--help)
|
|
114
|
+
pysec2pri ensembl labels (--help)
|
|
115
|
+
pysec2pri vgnc labels (--help)
|
|
103
116
|
```
|
|
104
117
|
|
|
105
|
-
|
|
106
|
-
mapping file in your current directory.
|
|
118
|
+
Or python:
|
|
107
119
|
|
|
108
|
-
To process local files and specify the output:
|
|
109
|
-
|
|
110
|
-
```bash
|
|
111
|
-
pysec2pri chebi ids ChEBI_complete_3star.sdf --output my_mappings.sssom.tsv
|
|
112
120
|
```
|
|
121
|
+
from pysec2pri import generate_ensembl_labels, generate_ensembl
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Replacing Ensembl by the supported database. These functions return either a
|
|
125
|
+
`IdMappingSet` or `LabelMappingSet`, SSSOM `MappingSet`s.
|
|
113
126
|
|
|
114
127
|
For more options and help on any command:
|
|
115
128
|
|
|
@@ -122,14 +135,23 @@ The default output is in [SSSOM](https://mapping-commons.github.io/sssom/)
|
|
|
122
135
|
|
|
123
136
|
### Updating IDs and labels
|
|
124
137
|
|
|
125
|
-
A generated mapping set can be used to update IDs and labels in Python:
|
|
138
|
+
A generated mapping set can be used to update IDs and labels in Python: ChEBI
|
|
139
|
+
synonyms:
|
|
126
140
|
|
|
127
141
|
```python
|
|
128
142
|
from pysec2pri import generate_chebi_synonyms, resolve_labels
|
|
129
|
-
|
|
143
|
+
chebi_ms = generate_chebi_synonyms()
|
|
130
144
|
resolve_labels(["Glucose", "ATP", "Guanine"], cs)
|
|
131
145
|
```
|
|
132
146
|
|
|
147
|
+
Ensembl gene identifiers in a dataframe:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
from pysec2pri import update_ids, generate_ensembl
|
|
151
|
+
ens_ms = generate_ensembl(version="115", species="9606")
|
|
152
|
+
df_with_new_column = update_ids(mapping_set=ens_ms, ids = df, at="Ensembl_id") # `at` is the name of the column
|
|
153
|
+
```
|
|
154
|
+
|
|
133
155
|
Or from the command line, given a TSV file `gene_ex.tsv`:
|
|
134
156
|
|
|
135
157
|
```
|
|
@@ -43,20 +43,33 @@ uv pip install git+https://github.com/jmillanacosta/pysec2pri.git
|
|
|
43
43
|
|
|
44
44
|
### Generating mapping sets
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
Mapping sets can be generated from bash:
|
|
47
47
|
|
|
48
48
|
```bash
|
|
49
|
-
|
|
49
|
+
# ID mapping sets
|
|
50
|
+
pysec2pri chebi ids (--help)
|
|
51
|
+
pysec2pri ensembl ids (--help)
|
|
52
|
+
pysec2pri hgnc ids (--help)
|
|
53
|
+
pysec2pri vgnc ids (--help)
|
|
54
|
+
pysec2pri ncbi ids (--help)
|
|
55
|
+
pysec2pri hmdb-protein ids (--help)
|
|
56
|
+
pysec2pri hmdb-gene ids (--help)
|
|
57
|
+
pysec2pri uniprot ids (--help)
|
|
58
|
+
# Label mapping sets
|
|
59
|
+
pysec2pri chebi labels (--help)
|
|
60
|
+
pysec2pri hgnc labels (--help)
|
|
61
|
+
pysec2pri ensembl labels (--help)
|
|
62
|
+
pysec2pri vgnc labels (--help)
|
|
50
63
|
```
|
|
51
64
|
|
|
52
|
-
|
|
53
|
-
mapping file in your current directory.
|
|
65
|
+
Or python:
|
|
54
66
|
|
|
55
|
-
To process local files and specify the output:
|
|
56
|
-
|
|
57
|
-
```bash
|
|
58
|
-
pysec2pri chebi ids ChEBI_complete_3star.sdf --output my_mappings.sssom.tsv
|
|
59
67
|
```
|
|
68
|
+
from pysec2pri import generate_ensembl_labels, generate_ensembl
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Replacing Ensembl by the supported database. These functions return either a
|
|
72
|
+
`IdMappingSet` or `LabelMappingSet`, SSSOM `MappingSet`s.
|
|
60
73
|
|
|
61
74
|
For more options and help on any command:
|
|
62
75
|
|
|
@@ -69,14 +82,23 @@ The default output is in [SSSOM](https://mapping-commons.github.io/sssom/)
|
|
|
69
82
|
|
|
70
83
|
### Updating IDs and labels
|
|
71
84
|
|
|
72
|
-
A generated mapping set can be used to update IDs and labels in Python:
|
|
85
|
+
A generated mapping set can be used to update IDs and labels in Python: ChEBI
|
|
86
|
+
synonyms:
|
|
73
87
|
|
|
74
88
|
```python
|
|
75
89
|
from pysec2pri import generate_chebi_synonyms, resolve_labels
|
|
76
|
-
|
|
90
|
+
chebi_ms = generate_chebi_synonyms()
|
|
77
91
|
resolve_labels(["Glucose", "ATP", "Guanine"], cs)
|
|
78
92
|
```
|
|
79
93
|
|
|
94
|
+
Ensembl gene identifiers in a dataframe:
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from pysec2pri import update_ids, generate_ensembl
|
|
98
|
+
ens_ms = generate_ensembl(version="115", species="9606")
|
|
99
|
+
df_with_new_column = update_ids(mapping_set=ens_ms, ids = df, at="Ensembl_id") # `at` is the name of the column
|
|
100
|
+
```
|
|
101
|
+
|
|
80
102
|
Or from the command line, given a TSV file `gene_ex.tsv`:
|
|
81
103
|
|
|
82
104
|
```
|
|
@@ -7,7 +7,7 @@ build-backend = "uv_build"
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "pysec2pri"
|
|
10
|
-
version = "0.2.
|
|
10
|
+
version = "0.2.2"
|
|
11
11
|
description = "Secondary to primary identifier mapping"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
authors = [
|
|
@@ -71,7 +71,7 @@ dependencies = [
|
|
|
71
71
|
"rdkit", # Chemistry toolkit for SDF parsing
|
|
72
72
|
"polars-runtime-32", # Polars runtime for 32-bit systems
|
|
73
73
|
"sssom-schema==1.1.0a5", # SSSOM schema for validation
|
|
74
|
-
"mapkgsutils>=0.0.
|
|
74
|
+
"mapkgsutils>=0.0.5",
|
|
75
75
|
]
|
|
76
76
|
|
|
77
77
|
# see https://peps.python.org/pep-0735/ and https://docs.astral.sh/uv/concepts/dependencies/#dependency-groups
|
|
@@ -255,7 +255,7 @@ known-first-party = [
|
|
|
255
255
|
docstring-code-format = true
|
|
256
256
|
|
|
257
257
|
[tool.bumpversion]
|
|
258
|
-
current_version = "0.2.
|
|
258
|
+
current_version = "0.2.2"
|
|
259
259
|
parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(?:-(?P<release>[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*))?(?:\\+(?P<build>[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*))?"
|
|
260
260
|
serialize = [
|
|
261
261
|
"{major}.{minor}.{patch}-{release}+{build}",
|
|
@@ -878,7 +878,7 @@ def _generate_ensembl_all_species(
|
|
|
878
878
|
version = check_ensembl_release().version
|
|
879
879
|
if version is None:
|
|
880
880
|
raise ValueError("Could not determine the latest Ensembl release.")
|
|
881
|
-
release_date = resolve_release_date("ensembl", version)
|
|
881
|
+
release_date = resolve_release_date("ensembl", version, species=9606)
|
|
882
882
|
|
|
883
883
|
species_list = discover_ensembl_species(version)
|
|
884
884
|
iterator: Any = species_list
|
|
@@ -1001,7 +1001,7 @@ def generate_ensembl_labels(
|
|
|
1001
1001
|
species: Canonical NCBI taxon ID, or ``"all"`` (default) to process
|
|
1002
1002
|
every species Ensembl publishes and combine them into one
|
|
1003
1003
|
mapping set -- see :func:`_generate_ensembl_all_species`. A
|
|
1004
|
-
symbol shared by two species becomes
|
|
1004
|
+
symbol shared by two species becomes ambiguous in
|
|
1005
1005
|
that combined output (there's no longer a per-species scope to
|
|
1006
1006
|
disambiguate it). This is network-heavy (~276 species); pass
|
|
1007
1007
|
an explicit taxon ID for a fast, single-species run.
|
|
@@ -464,7 +464,7 @@ def ensembl_label_history_cmd(
|
|
|
464
464
|
Ensembl's core schema has no previous-gene-symbol table, so this walks
|
|
465
465
|
every historical release (or a bounded --from-version/--to-version
|
|
466
466
|
range) and diffs each release's primary-label snapshot to recover
|
|
467
|
-
|
|
467
|
+
true previous -> current symbol transitions. Network-heavy and
|
|
468
468
|
resumable; run on demand, not part of normal mapping generation.
|
|
469
469
|
"""
|
|
470
470
|
from pysec2pri.api import generate_ensembl_label_history
|
|
@@ -643,6 +643,7 @@ def _make_consolidate_cmd(cfg_id: str, extra_opts: list[Callable[..., Any]]) ->
|
|
|
643
643
|
cache_dir: Path | None,
|
|
644
644
|
force: bool,
|
|
645
645
|
no_progress: bool,
|
|
646
|
+
output: Path | None,
|
|
646
647
|
**extra_kwargs: Any,
|
|
647
648
|
) -> None:
|
|
648
649
|
from pysec2pri.consolidate import _sssom_output_path, consolidate_mapping_dates
|
|
@@ -653,18 +654,19 @@ def _make_consolidate_cmd(cfg_id: str, extra_opts: list[Callable[..., Any]]) ->
|
|
|
653
654
|
f"(mode={mode}{', ' + extras if extras else ''}, {mapping_sets})..."
|
|
654
655
|
)
|
|
655
656
|
try:
|
|
656
|
-
path = consolidate_mapping_dates(
|
|
657
|
+
path, _ = consolidate_mapping_dates(
|
|
657
658
|
cfg_id,
|
|
658
659
|
mode=mode,
|
|
659
660
|
cache_dir=cache_dir,
|
|
660
661
|
mapping_sets=mapping_sets,
|
|
661
662
|
show_progress=not no_progress,
|
|
662
663
|
force=force,
|
|
664
|
+
output=output,
|
|
663
665
|
**extra_kwargs,
|
|
664
666
|
)
|
|
665
667
|
except ValueError as exc:
|
|
666
668
|
raise click.ClickException(str(exc)) from exc
|
|
667
|
-
click.echo(f"Wrote consolidated mapping set -> {_sssom_output_path(path)}")
|
|
669
|
+
click.echo(f"Wrote consolidated mapping set -> {output or _sssom_output_path(path)}")
|
|
668
670
|
|
|
669
671
|
decorators: list[Callable[..., Any]] = [
|
|
670
672
|
click.option(
|
|
@@ -693,6 +695,13 @@ def _make_consolidate_cmd(cfg_id: str, extra_opts: list[Callable[..., Any]]) ->
|
|
|
693
695
|
default=False,
|
|
694
696
|
help="Re-scan every release, ignoring resume state.",
|
|
695
697
|
),
|
|
698
|
+
click.option(
|
|
699
|
+
"-o",
|
|
700
|
+
"--output",
|
|
701
|
+
type=PathType,
|
|
702
|
+
default=None,
|
|
703
|
+
help="Also write the consolidated SSSOM mapping set to this path.",
|
|
704
|
+
),
|
|
696
705
|
_opt_no_progress,
|
|
697
706
|
]
|
|
698
707
|
for dec in reversed(decorators):
|
|
@@ -7,13 +7,6 @@ prefix: "ENSEMBL"
|
|
|
7
7
|
curie_base_url: "http://identifiers.org/ensembl/"
|
|
8
8
|
entity_types: ["genes"]
|
|
9
9
|
|
|
10
|
-
# Canonical key = NCBI taxon ID. `token` is Ensembl's own species directory
|
|
11
|
-
# component (e.g. https://ftp.ensembl.org/pub/release-115/mysql/homo_sapiens_core_115_38/).
|
|
12
|
-
# `default: "all"` processes every species Ensembl publishes for the
|
|
13
|
-
# release and combines them into one mapping set (see
|
|
14
|
-
# pysec2pri.api._generate_ensembl_all_species) -- network-heavy (~276
|
|
15
|
-
# species, one download each); pass an explicit taxon ID for a fast,
|
|
16
|
-
# single-species run.
|
|
17
10
|
species:
|
|
18
11
|
default: "all"
|
|
19
12
|
available:
|
|
@@ -9,14 +9,6 @@ prefix: "NCBIGene"
|
|
|
9
9
|
curie_base_url: "http://identifiers.org/ncbigene/"
|
|
10
10
|
entity_types: ["genes"]
|
|
11
11
|
|
|
12
|
-
# NCBI's own token IS the canonical taxon ID (no translation needed, unlike
|
|
13
|
-
# Ensembl) -- gene_history/gene_info are filtered directly on #tax_id, which
|
|
14
|
-
# accepts ANY taxon ID, not just the ones listed here. `available` exists
|
|
15
|
-
# only to drive CLI help/choices for common species; NCBIParser never calls
|
|
16
|
-
# species_token(), so it never restricts which taxon IDs are accepted.
|
|
17
|
-
# `default: "all"` processes every organism in the file instead of
|
|
18
|
-
# filtering to one (see NCBIParser.ALL_SPECIES) -- cheap here since
|
|
19
|
-
# gene_info/gene_history are single global files already being downloaded.
|
|
20
12
|
species:
|
|
21
13
|
default: "all"
|
|
22
14
|
available:
|
|
@@ -7,22 +7,6 @@ prefix: "VGNC"
|
|
|
7
7
|
curie_base_url: "http://identifiers.org/vgnc/"
|
|
8
8
|
entity_types: ["genes"]
|
|
9
9
|
|
|
10
|
-
# VGNC (Vertebrate Gene Nomenclature Committee) names genes for vertebrate
|
|
11
|
-
# species that lack their own nomenclature committee (human/mouse/rat are
|
|
12
|
-
# deliberately out of scope -- HGNC/MGI/RGD cover those). Unlike Ensembl
|
|
13
|
-
# (one download URL per species), VGNC publishes a single global TSV per
|
|
14
|
-
# file with a `taxon_id` column, filtered the same way NCBI's
|
|
15
|
-
# gene_info/gene_history are filtered by `#tax_id` -- see VGNCParser.
|
|
16
|
-
# `default: "all"` processes every species together (see
|
|
17
|
-
# VGNCParser.ALL_SPECIES) -- cheap for ids (the global withdrawn set is
|
|
18
|
-
# always parsed in full regardless), heavier for labels (a symbol shared
|
|
19
|
-
# by two species becomes genuinely ambiguous with no per-species scope to
|
|
20
|
-
# disambiguate it). VGNCParser never calls species_token(), so `available`
|
|
21
|
-
# exists only to drive CLI help/choices and never restricts which taxon IDs
|
|
22
|
-
# are accepted. Taxon IDs below were verified against the live data (each
|
|
23
|
-
# species' own vgnc/tsv/<species>/*_vgnc_gene_set_All.tsv, cross-referenced
|
|
24
|
-
# by VGNC ID against vgnc/tsv/all/all_vgnc_gene_set_All.tsv's taxon_id
|
|
25
|
-
# column).
|
|
26
10
|
species:
|
|
27
11
|
default: "all"
|
|
28
12
|
available:
|
|
@@ -161,6 +161,59 @@ _read_meta = _consolidate.read_meta
|
|
|
161
161
|
_write_meta = _consolidate.write_meta
|
|
162
162
|
|
|
163
163
|
|
|
164
|
+
def _save_optional_output(mapping_set: BaseMappingSet, output: Path | None) -> None:
|
|
165
|
+
"""Write *mapping_set* to *output* as SSSOM when an output path is given."""
|
|
166
|
+
if output is not None:
|
|
167
|
+
mapping_set.save("sssom", output)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _consolidate_by_date_full(
|
|
171
|
+
cache_path: Path,
|
|
172
|
+
meta_path: Path,
|
|
173
|
+
*,
|
|
174
|
+
run_one_version: Callable[[], Any],
|
|
175
|
+
) -> bool:
|
|
176
|
+
"""Date mode that caches the *full* mapping set, not just dated rows.
|
|
177
|
+
|
|
178
|
+
:func:`mapkgsutils.consolidate.consolidate_by_date` keeps only rows that
|
|
179
|
+
carry a real per-row ``mapping_date`` and drops the rest, so a single
|
|
180
|
+
parse of e.g. HGNC labels would cache only its ~12k dated previous-symbol
|
|
181
|
+
mappings and lose the ~48k alias mappings. This builds a complete cache
|
|
182
|
+
from one (current) parse: every mapping is kept, stamped with its own
|
|
183
|
+
per-row date when present and left undated otherwise (exactly how the
|
|
184
|
+
release walk treats an unresolvable date).
|
|
185
|
+
|
|
186
|
+
Returns ``True`` (cache written) when the parse yields at least one real
|
|
187
|
+
per-row date, and ``False`` (cache untouched) when it yields none -- so
|
|
188
|
+
date-less sources (ChEBI, UniProt) still fall back to a release walk
|
|
189
|
+
rather than caching a fully undated set.
|
|
190
|
+
"""
|
|
191
|
+
mapping_set = run_one_version()
|
|
192
|
+
mappings = mapping_set.mappings or []
|
|
193
|
+
if not any(getattr(m, "mapping_date", None) for m in mappings):
|
|
194
|
+
return False
|
|
195
|
+
|
|
196
|
+
version_label = str(getattr(mapping_set, "mapping_set_version", None) or "current")
|
|
197
|
+
records: dict[str, dict[str, str]] = {}
|
|
198
|
+
for m in mappings:
|
|
199
|
+
# record_id is release-scoped; its trailing 16 hex chars are the
|
|
200
|
+
# version-independent pair hash (see mapkgsutils.parsers.base.pair_hash).
|
|
201
|
+
pair_key = str(getattr(m, "record_id", None) or "")[-16:]
|
|
202
|
+
if not pair_key:
|
|
203
|
+
continue
|
|
204
|
+
date_str = str(m.mapping_date) if getattr(m, "mapping_date", None) else ""
|
|
205
|
+
records[pair_key] = {
|
|
206
|
+
"first_seen_version": version_label,
|
|
207
|
+
"first_seen_date": date_str,
|
|
208
|
+
"last_seen_version": version_label,
|
|
209
|
+
"last_seen_date": date_str,
|
|
210
|
+
"fields_json": _consolidate._mapping_fields_json(m),
|
|
211
|
+
}
|
|
212
|
+
_write_cache(cache_path, records)
|
|
213
|
+
_write_meta(meta_path, version_label)
|
|
214
|
+
return True
|
|
215
|
+
|
|
216
|
+
|
|
164
217
|
def load_mapping_dates(
|
|
165
218
|
datasource: str,
|
|
166
219
|
cache_dir: Path | None = None,
|
|
@@ -359,8 +412,12 @@ def _build_consolidated_mapping_set(
|
|
|
359
412
|
|
|
360
413
|
def _write_consolidated_sssom(
|
|
361
414
|
datasource: str, mapping_sets: str, cache_path: Path, meta_path: Path
|
|
362
|
-
) -> Path:
|
|
363
|
-
"""Build and save the companion SSSOM mapping set next to the cache file.
|
|
415
|
+
) -> tuple[Path, Any]:
|
|
416
|
+
"""Build and save the companion SSSOM mapping set next to the cache file.
|
|
417
|
+
|
|
418
|
+
Returns ``(output_path, mapping_set)`` -- see
|
|
419
|
+
:func:`mapkgsutils.consolidate.write_consolidated_sssom`.
|
|
420
|
+
"""
|
|
364
421
|
from pysec2pri.parsers.base import IdMappingSet, LabelMappingSet
|
|
365
422
|
|
|
366
423
|
config = ALL_DATASOURCES[datasource]
|
|
@@ -383,8 +440,9 @@ def consolidate_mapping_dates(
|
|
|
383
440
|
mapping_sets: str = "ids",
|
|
384
441
|
show_progress: bool = True,
|
|
385
442
|
force: bool = False,
|
|
443
|
+
output: Path | None = None,
|
|
386
444
|
**kwargs: Any,
|
|
387
|
-
) -> Path:
|
|
445
|
+
) -> tuple[Path, BaseMappingSet]:
|
|
388
446
|
"""Build/update the first-seen-date index for *datasource*.
|
|
389
447
|
|
|
390
448
|
Two strategies, selected via *mode*:
|
|
@@ -393,17 +451,18 @@ def consolidate_mapping_dates(
|
|
|
393
451
|
(oldest first, resuming from the last completed version unless
|
|
394
452
|
*force* is set). For every mapping seen, records the version/date it
|
|
395
453
|
first appeared and keeps bumping the version/date it was last seen.
|
|
396
|
-
This is a slow, network-heavy operation (~250 releases for ChEBI) and
|
|
454
|
+
This is a slow, network-heavy operation (e.g. ~250 releases for ChEBI) and
|
|
397
455
|
is meant to be run manually/as a one-off, not as part of normal
|
|
398
456
|
mapping generation. Requires a versioned archive (ChEBI, HGNC,
|
|
399
457
|
UniProt); raises :class:`ValueError` for datasources without one
|
|
400
458
|
(e.g. NCBI).
|
|
401
|
-
- ``"date"``: a single (current) parse,
|
|
402
|
-
per-row ``mapping_date`` the datasource's parser
|
|
403
|
-
(e.g. HGNC's ``date_symbol_changed``, NCBI's
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
459
|
+
- ``"date"``: a single (current) parse, caching the *full* mapping set
|
|
460
|
+
with whatever real per-row ``mapping_date`` the datasource's parser
|
|
461
|
+
already produces (e.g. HGNC's ``date_symbol_changed``, NCBI's
|
|
462
|
+
``Discontinue_Date``); rows without such a date are still cached, just
|
|
463
|
+
left undated. Fast -- no historical walk. If *datasource* never
|
|
464
|
+
produces a real per-row date at all (ChEBI, UniProt), falls back to
|
|
465
|
+
``"release"`` mode and warns the user.
|
|
407
466
|
|
|
408
467
|
Either way, alongside the internal cache file this also writes a
|
|
409
468
|
companion real SSSOM mapping set (see :func:`_sssom_output_path`) where
|
|
@@ -420,13 +479,18 @@ def consolidate_mapping_dates(
|
|
|
420
479
|
(``"release"`` mode only).
|
|
421
480
|
force: Re-scan every release from scratch, ignoring any existing
|
|
422
481
|
cache/resume state (``"release"`` mode only).
|
|
482
|
+
output: Optional path to also write the consolidated SSSOM mapping
|
|
483
|
+
set to. The internal cache-adjacent copy (see
|
|
484
|
+
:func:`_sssom_output_path`) is always written; when *output* is
|
|
485
|
+
given, the same full mapping set is additionally saved there.
|
|
423
486
|
**kwargs: Datasource-specific knobs (``subset`` for ChEBI, ``species``
|
|
424
487
|
for NCBI/Ensembl); ignored for datasources with no such config
|
|
425
488
|
block.
|
|
426
489
|
|
|
427
490
|
Returns:
|
|
428
|
-
|
|
429
|
-
the companion SSSOM mapping set
|
|
491
|
+
``(cache_path, mapping_set)``: the path to the written cache TSV
|
|
492
|
+
(see :func:`_sssom_output_path` for the companion SSSOM mapping set
|
|
493
|
+
written alongside it) and the in-memory consolidated mapping set.
|
|
430
494
|
|
|
431
495
|
Raises:
|
|
432
496
|
ValueError: For an unknown *mode*, an unsupported *datasource*, an
|
|
@@ -461,14 +525,17 @@ def consolidate_mapping_dates(
|
|
|
461
525
|
meta_path = _meta_path(cache_dir, datasource, mapping_sets, **kwargs)
|
|
462
526
|
|
|
463
527
|
if mode == "date":
|
|
464
|
-
got_dates =
|
|
528
|
+
got_dates = _consolidate_by_date_full(
|
|
465
529
|
cache_path,
|
|
466
530
|
meta_path,
|
|
467
531
|
run_one_version=lambda: _run_one_version(datasource, None, mapping_sets, **kwargs),
|
|
468
532
|
)
|
|
469
533
|
if got_dates:
|
|
470
|
-
|
|
471
|
-
|
|
534
|
+
_, mapping_set = _write_consolidated_sssom(
|
|
535
|
+
datasource, mapping_sets, cache_path, meta_path
|
|
536
|
+
)
|
|
537
|
+
_save_optional_output(mapping_set, output)
|
|
538
|
+
return cache_path, mapping_set
|
|
472
539
|
msg = (
|
|
473
540
|
f"{datasource!r} has no parseable per-row mapping dates; "
|
|
474
541
|
"falling back to mode='release'."
|
|
@@ -492,12 +559,13 @@ def consolidate_mapping_dates(
|
|
|
492
559
|
show_progress=show_progress,
|
|
493
560
|
force=force,
|
|
494
561
|
)
|
|
495
|
-
_write_consolidated_sssom(datasource, mapping_sets, cache_path, meta_path)
|
|
496
|
-
|
|
562
|
+
_, mapping_set = _write_consolidated_sssom(datasource, mapping_sets, cache_path, meta_path)
|
|
563
|
+
_save_optional_output(mapping_set, output)
|
|
564
|
+
return cache_path, mapping_set
|
|
497
565
|
|
|
498
566
|
|
|
499
567
|
# Cross-release label history (e.g. Ensembl, whose core schema has no
|
|
500
|
-
# previous-gene-symbol table
|
|
568
|
+
# previous-gene-symbol table: previous->current symbol transitions
|
|
501
569
|
# are recovered by diffing each release's current-label snapshot).
|
|
502
570
|
|
|
503
571
|
|
|
@@ -17,7 +17,6 @@ import gzip
|
|
|
17
17
|
import io
|
|
18
18
|
import re
|
|
19
19
|
import zipfile
|
|
20
|
-
from collections.abc import Callable
|
|
21
20
|
from pathlib import Path
|
|
22
21
|
from typing import IO, TYPE_CHECKING
|
|
23
22
|
|
|
@@ -85,7 +84,6 @@ class HMDBParser(BaseParser):
|
|
|
85
84
|
element_tag: str,
|
|
86
85
|
prefix: str,
|
|
87
86
|
desc: str,
|
|
88
|
-
secondary_normaliser: Callable[[str], str] | None = None,
|
|
89
87
|
) -> BaseMappingSet:
|
|
90
88
|
"""Parse an HMDB XML file and return a mapping set.
|
|
91
89
|
|
|
@@ -95,8 +93,6 @@ class HMDBParser(BaseParser):
|
|
|
95
93
|
``"protein"``).
|
|
96
94
|
prefix: CURIE prefix (``"HMDB"`` for both record types).
|
|
97
95
|
desc: Progress-bar description string.
|
|
98
|
-
secondary_normaliser: Optional callable ``(raw_str) -> str``
|
|
99
|
-
applied to each raw secondary accession before prefixing.
|
|
100
96
|
|
|
101
97
|
Returns:
|
|
102
98
|
:class:`BaseMappingSet` with computed cardinalities and
|
|
@@ -130,10 +126,10 @@ class HMDBParser(BaseParser):
|
|
|
130
126
|
accession_elem = elem.find("accession")
|
|
131
127
|
if accession_elem is not None and accession_elem.text:
|
|
132
128
|
primary_ids_found.add(f"{prefix}:{accession_elem.text.strip()}")
|
|
133
|
-
rows_data.extend(self._process_record(elem, prefix
|
|
129
|
+
rows_data.extend(self._process_record(elem, prefix))
|
|
134
130
|
elem.clear()
|
|
135
131
|
except DefusedET.ParseError:
|
|
136
|
-
rows_data = self._parse_simple_xml(file_path, element_tag, prefix
|
|
132
|
+
rows_data = self._parse_simple_xml(file_path, element_tag, prefix)
|
|
137
133
|
|
|
138
134
|
mappings = self._build_mappings(
|
|
139
135
|
rows_data, fixed, desc="Building HMDB mappings", total=len(rows_data)
|
|
@@ -147,9 +143,14 @@ class HMDBParser(BaseParser):
|
|
|
147
143
|
self,
|
|
148
144
|
elem: Element,
|
|
149
145
|
prefix: str,
|
|
150
|
-
secondary_normaliser: Callable[[str], str] | None,
|
|
151
146
|
) -> list[dict[str, str]]:
|
|
152
|
-
"""Extract row dicts from a single record element.
|
|
147
|
+
"""Extract row dicts from a single record element.
|
|
148
|
+
|
|
149
|
+
Bare-numeric ``secondary_accessions`` entries are skipped: in the
|
|
150
|
+
proteins file these are HMDB's internal database row ids (not legacy
|
|
151
|
+
accessions), and a bare integer is never a valid HMDB accession for
|
|
152
|
+
either record type. See issue #44.
|
|
153
|
+
"""
|
|
153
154
|
# Primary accession
|
|
154
155
|
accession_elem = elem.find("hmdb:accession", HMDB_NS)
|
|
155
156
|
if accession_elem is None:
|
|
@@ -177,8 +178,8 @@ class HMDBParser(BaseParser):
|
|
|
177
178
|
if not sec_elem.text:
|
|
178
179
|
continue
|
|
179
180
|
raw_sec = sec_elem.text.strip()
|
|
180
|
-
if
|
|
181
|
-
|
|
181
|
+
if _BARE_NUM_RE.match(raw_sec):
|
|
182
|
+
continue
|
|
182
183
|
secondary_id = f"{prefix}:{raw_sec}"
|
|
183
184
|
rows.append(
|
|
184
185
|
{
|
|
@@ -200,7 +201,6 @@ class HMDBParser(BaseParser):
|
|
|
200
201
|
file_path: Path,
|
|
201
202
|
element_tag: str,
|
|
202
203
|
prefix: str,
|
|
203
|
-
secondary_normaliser: Callable[[str], str] | None,
|
|
204
204
|
) -> list[dict[str, str]]:
|
|
205
205
|
"""Fallback parser for test files without a namespace declaration."""
|
|
206
206
|
rows_data: list[dict[str, str]] = []
|
|
@@ -208,7 +208,7 @@ class HMDBParser(BaseParser):
|
|
|
208
208
|
tree = DefusedET.parse(file_path)
|
|
209
209
|
root = tree.getroot()
|
|
210
210
|
for record in root.findall(f".//{element_tag}"):
|
|
211
|
-
rows_data.extend(self._process_record(record, prefix
|
|
211
|
+
rows_data.extend(self._process_record(record, prefix))
|
|
212
212
|
except Exception as e:
|
|
213
213
|
logger.warning("Failed to parse HMDB XML: %s", e)
|
|
214
214
|
return rows_data
|
|
@@ -329,9 +329,15 @@ class HMDBProteinParser(HMDBParser):
|
|
|
329
329
|
|
|
330
330
|
Reads configuration from ``hmdb_proteins.yaml``.
|
|
331
331
|
Primary accessions have the form ``HMDBP…`` (e.g. ``HMDBP00001``).
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
:
|
|
332
|
+
|
|
333
|
+
A record's ``<secondary_accessions>`` block holds two distinct kinds of
|
|
334
|
+
value. Only the first is a real identifier:
|
|
335
|
+
|
|
336
|
+
* **``HMDBP``-prefixed accessions** (e.g. ``HMDBP05261``): genuine retired
|
|
337
|
+
accessions that map to the record's primary accession.
|
|
338
|
+
* **Bare numeric values** (e.g. ``5229``): HMDB's internal database row id,
|
|
339
|
+
*not* a legacy accession. These are dropped by the shared
|
|
340
|
+
:meth:`~HMDBParser._process_record` (see issue #44).
|
|
335
341
|
"""
|
|
336
342
|
|
|
337
343
|
datasource_name = "hmdb_proteins"
|
|
@@ -341,26 +347,6 @@ class HMDBProteinParser(HMDBParser):
|
|
|
341
347
|
"""Proteins download URL from ``hmdb_proteins.yaml``."""
|
|
342
348
|
return self.get_download_url("proteins") or ""
|
|
343
349
|
|
|
344
|
-
@staticmethod
|
|
345
|
-
def _normalise_protein_secondary(raw: str) -> str:
|
|
346
|
-
"""Normalise a raw secondary protein accession to a ``HMDBP``-prefixed form.
|
|
347
|
-
|
|
348
|
-
Bare numeric values (legacy IDs) are zero-padded to five digits and
|
|
349
|
-
prefixed with ``HMDBP`` to match the primary accession pattern.
|
|
350
|
-
Already-prefixed values are returned unchanged.
|
|
351
|
-
|
|
352
|
-
Args:
|
|
353
|
-
raw: Raw secondary accession string (e.g. ``"5229"`` or
|
|
354
|
-
``"HMDBP05229"``).
|
|
355
|
-
|
|
356
|
-
Returns:
|
|
357
|
-
Normalised accession string (e.g. ``"HMDBP05229"``).
|
|
358
|
-
"""
|
|
359
|
-
raw = raw.strip()
|
|
360
|
-
if _BARE_NUM_RE.match(raw):
|
|
361
|
-
return f"HMDBP{int(raw):05d}"
|
|
362
|
-
return raw
|
|
363
|
-
|
|
364
350
|
def parse(self, input_path: Path | str | None) -> BaseMappingSet:
|
|
365
351
|
"""Parse ``hmdb_proteins.xml`` (or ``.zip`` / ``.gz``).
|
|
366
352
|
|
|
@@ -377,7 +363,6 @@ class HMDBProteinParser(HMDBParser):
|
|
|
377
363
|
element_tag="protein",
|
|
378
364
|
prefix="HMDBP",
|
|
379
365
|
desc="Parsing HMDB proteins XML",
|
|
380
|
-
secondary_normaliser=self._normalise_protein_secondary,
|
|
381
366
|
)
|
|
382
367
|
|
|
383
368
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|