altar-conservation 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- altar_conservation-0.3.0/.gitignore +33 -0
- altar_conservation-0.3.0/LICENSE +21 -0
- altar_conservation-0.3.0/PKG-INFO +128 -0
- altar_conservation-0.3.0/README.md +103 -0
- altar_conservation-0.3.0/altar_conservation/__init__.py +15 -0
- altar_conservation-0.3.0/altar_conservation/build_dataset.py +319 -0
- altar_conservation-0.3.0/altar_conservation/py.typed +0 -0
- altar_conservation-0.3.0/altar_conservation/source.py +164 -0
- altar_conservation-0.3.0/altar_conservation/tracks.py +87 -0
- altar_conservation-0.3.0/hf-card/README.md +114 -0
- altar_conservation-0.3.0/pyproject.toml +56 -0
- altar_conservation-0.3.0/tests/test_conservation.py +477 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
*.pyc
|
|
2
|
+
*~
|
|
3
|
+
**/__pycache__/*
|
|
4
|
+
*.swp
|
|
5
|
+
.vscode/
|
|
6
|
+
.idea/
|
|
7
|
+
.DS_Store
|
|
8
|
+
.env*
|
|
9
|
+
.mypy_cache/
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
.pytest_cache/
|
|
12
|
+
.coverage
|
|
13
|
+
htmlcov/
|
|
14
|
+
coverage.xml
|
|
15
|
+
pytest.xml
|
|
16
|
+
.hypothesis/
|
|
17
|
+
.python-version
|
|
18
|
+
.venv
|
|
19
|
+
.venv-*/
|
|
20
|
+
*.egg-info/
|
|
21
|
+
build/
|
|
22
|
+
dist/
|
|
23
|
+
site/
|
|
24
|
+
*.sqlite3
|
|
25
|
+
*.log
|
|
26
|
+
|
|
27
|
+
# Generated variant indexes and datasets. The small canonical gene table is tracked.
|
|
28
|
+
altar/altar/variants/data/ccres.dnatree
|
|
29
|
+
altar/altar/variants/data/region_annotations.parquet
|
|
30
|
+
altar/altar/variants/data/variants.pkl.gz
|
|
31
|
+
altar/altar/variants/data/region_provenance.json
|
|
32
|
+
altar/altar/variants/data/raw/
|
|
33
|
+
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Riya Sinha
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: altar-conservation
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Base-wise conservation score tracks (UCSC phyloP) for Altar
|
|
5
|
+
Project-URL: Documentation, https://kundajelab.github.io/altar/
|
|
6
|
+
Author: Riya Sinha
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Requires-Dist: altar[gcs]<0.4,>=0.3.0
|
|
19
|
+
Provides-Extra: build
|
|
20
|
+
Requires-Dist: pybigtools>=0.3.0; extra == 'build'
|
|
21
|
+
Provides-Extra: test
|
|
22
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'test'
|
|
23
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# altar-conservation
|
|
27
|
+
|
|
28
|
+
This binding adds base-wise conservation scores to variants. Its one track today is UCSC's hg38 phyloP 100-way
|
|
29
|
+
(`hg38.phyloP100way.bw`, May 2015): `phylop100way`, an annotation source (`PhyloP100wayAnnotationSource`), adds a
|
|
30
|
+
`phylop100way` column, the phyloP score at the variant's base, from a Parquet copy of the track: the one the
|
|
31
|
+
Kundaje lab publishes, or one you own. phyloP is a signed -log10 p-value: positive where a base evolves more slowly
|
|
32
|
+
than neutral across 100 vertebrates (conserved), negative where it evolves faster.
|
|
33
|
+
|
|
34
|
+
## Annotate variants from the published copy
|
|
35
|
+
|
|
36
|
+
The Kundaje lab publishes the track in Altar's position score layout (`PHYLOP100WAY.published`):
|
|
37
|
+
|
|
38
|
+
| Location | URI |
|
|
39
|
+
| --- | --- |
|
|
40
|
+
| `gcs` (default) | `gs://kundajelab-altar/conservation/phylop100way-hg38` |
|
|
41
|
+
| `hf` | `hf://datasets/altar-genomics/phylop100way-hg38` ([dataset card](https://huggingface.co/datasets/altar-genomics/phylop100way-hg38)) |
|
|
42
|
+
|
|
43
|
+
A run spec reads it with no options; the package depends on `altar[gcs]`, and the `hf` mirror needs `altar[hf]`:
|
|
44
|
+
|
|
45
|
+
```toml
|
|
46
|
+
[sources.phylop]
|
|
47
|
+
type = "phylop100way"
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Hugging Face limits anonymous requests, so a large cohort read with `location = "hf"` can stall; read the bucket, or
|
|
51
|
+
pull a copy and name it with `dataset` ([published locations](https://kundajelab.github.io/altar/guides/published-datasets/#published-locations)).
|
|
52
|
+
|
|
53
|
+
In Python:
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
import asyncio
|
|
57
|
+
|
|
58
|
+
from altar_conservation import PhyloP100wayAnnotationSource
|
|
59
|
+
|
|
60
|
+
source = PhyloP100wayAnnotationSource.from_options() # the published copy
|
|
61
|
+
rows = asyncio.run(source.annotate(["chr7:140753336:A:T"]))
|
|
62
|
+
rows["chr7:140753336:A:T"]["phylop100way"] # 9.236
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The contract is `org.kundajelab.altar.annotation.phylop100way` (version 1.0.0, `PHYLOP100WAY_ANNOTATIONS`), one
|
|
66
|
+
float column `phylop100way`. The copy holds 2,837,915,604 scored bases of chr1–22, X, Y and M; a variant at an
|
|
67
|
+
unscored base, or on any other contig (alternate, unlocalized or unplaced), has no row.
|
|
68
|
+
|
|
69
|
+
`altar pull phylop100way --to <dir>` copies the published dataset (5.0 GB), checking each file's release; name the
|
|
70
|
+
copy with `dataset`. The options:
|
|
71
|
+
|
|
72
|
+
| Option | Meaning |
|
|
73
|
+
| --- | --- |
|
|
74
|
+
| `dataset` | An owned copy, as a path or a URI: a pulled copy, or one built with the builder below. |
|
|
75
|
+
| `location` | `gcs` (default) or `hf`, for the published copy. |
|
|
76
|
+
| `connection`, `storage_options` | How to reach a bucket. |
|
|
77
|
+
| `indels` | `skip` (default) scores SNVs only; `max` scores every variant (below). |
|
|
78
|
+
| `name` | Rename the source. |
|
|
79
|
+
| `genome_build` | Optional; must be `hg38`. |
|
|
80
|
+
| `prioritize_predicate` | In code only: a predicate over `phylop100way` that gates `prioritized`. |
|
|
81
|
+
|
|
82
|
+
`table` is not read yet: BigQuery would scan the 2.8-billion-row table for every chunk of variants, so the source
|
|
83
|
+
refuses it, before any other option, and asks for a pulled copy. An unknown option or `location` raises a
|
|
84
|
+
`ValueError` that lists the valid ones.
|
|
85
|
+
|
|
86
|
+
The track is on hg38 only. `altar run` stops before ingesting anything when the spec's `[run] genome_build` is
|
|
87
|
+
another assembly, naming the source.
|
|
88
|
+
|
|
89
|
+
With `indels = "max"`, a variant that is not an SNV is scored by the greatest score over the bases it touches, as
|
|
90
|
+
VEP takes them: a deletion's deleted bases (after the padding base), the two bases an insertion falls between, or
|
|
91
|
+
any other change's REF span. Unscored bases are ignored; a variant with none scored has no row. SNVs score the same
|
|
92
|
+
either way. The identity is the contract with release `2015-05` and `hg38`; with `indels = "max"` the release is
|
|
93
|
+
`2015-05+indels-max`, since it scores variants the default leaves out.
|
|
94
|
+
|
|
95
|
+
## Build the dataset
|
|
96
|
+
|
|
97
|
+
```console
|
|
98
|
+
pip install "altar-conservation[build]"
|
|
99
|
+
curl -O https://hgdownload.soe.ucsc.edu/goldenPath/hg38/phyloP100way/hg38.phyloP100way.bw
|
|
100
|
+
altar-conservation-build --track phylop100way --input hg38.phyloP100way.bw --output /scratch/phylop100way-hg38
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The `build` extra installs [pybigtools](https://github.com/jackh726/bigtools), which has wheels for Linux and
|
|
104
|
+
macOS. The builder checks the file's MD5 against UCSC's `md5sum.txt`, then converts each primary contig in its own
|
|
105
|
+
process (`--workers`, default one per contig up to the CPU count): it reads the contig's records, turns UCSC's 0-based
|
|
106
|
+
intervals into one row per base with a 1-based `pos`, checks that positions strictly increase, and writes
|
|
107
|
+
`chr=<contig>/scores.parquet` with `PartitionedWriter` in 128,000-row groups, `pos` delta-encoded, with the track,
|
|
108
|
+
release and genome build in every footer. Other contigs are skipped. On a 64-core server the build took 72 s
|
|
109
|
+
(15 s of it the MD5) and about 14 GB of memory across 25 processes, about 0.7 GB each. The output is a local
|
|
110
|
+
directory; copy it to a bucket yourself.
|
|
111
|
+
|
|
112
|
+
## Data terms and citation
|
|
113
|
+
|
|
114
|
+
UCSC's phyloP files are "freely available for public use"; UCSC asks you to credit the data and cite the Genome
|
|
115
|
+
Browser. Please cite:
|
|
116
|
+
|
|
117
|
+
- Pollard KS, Hubisz MJ, Rosenbloom KR, Siepel A. Detection of nonneutral substitution rates on mammalian phylogenies. *Genome
|
|
118
|
+
Research* 2010;20(1):110–121. doi:10.1101/gr.097857.109.
|
|
119
|
+
- Siepel A, et al. Evolutionarily conserved elements in vertebrate, insect, worm, and yeast genomes. *Genome
|
|
120
|
+
Research* 2005;15(8):1034–1050. doi:10.1101/gr.3715005.
|
|
121
|
+
- The UCSC Genome Browser's latest database paper (<https://genome.ucsc.edu/cite.html>): Casper J, et al.
|
|
122
|
+
*Nucleic Acids Research* 2026. PMID 41251146.
|
|
123
|
+
|
|
124
|
+
[Cite these data](https://kundajelab.github.io/altar/integrations/evidence/#cite-these-data) lists these with the
|
|
125
|
+
other datasets Altar publishes.
|
|
126
|
+
|
|
127
|
+
The binding ships no data. Its contract is its own; the source and layout come from `altar`, and the binding is
|
|
128
|
+
tested against Altar's conformance suites.
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# altar-conservation
|
|
2
|
+
|
|
3
|
+
This binding adds base-wise conservation scores to variants. Its one track today is UCSC's hg38 phyloP 100-way
|
|
4
|
+
(`hg38.phyloP100way.bw`, May 2015): `phylop100way`, an annotation source (`PhyloP100wayAnnotationSource`), adds a
|
|
5
|
+
`phylop100way` column, the phyloP score at the variant's base, from a Parquet copy of the track: the one the
|
|
6
|
+
Kundaje lab publishes, or one you own. phyloP is a signed -log10 p-value: positive where a base evolves more slowly
|
|
7
|
+
than neutral across 100 vertebrates (conserved), negative where it evolves faster.
|
|
8
|
+
|
|
9
|
+
## Annotate variants from the published copy
|
|
10
|
+
|
|
11
|
+
The Kundaje lab publishes the track in Altar's position score layout (`PHYLOP100WAY.published`):
|
|
12
|
+
|
|
13
|
+
| Location | URI |
|
|
14
|
+
| --- | --- |
|
|
15
|
+
| `gcs` (default) | `gs://kundajelab-altar/conservation/phylop100way-hg38` |
|
|
16
|
+
| `hf` | `hf://datasets/altar-genomics/phylop100way-hg38` ([dataset card](https://huggingface.co/datasets/altar-genomics/phylop100way-hg38)) |
|
|
17
|
+
|
|
18
|
+
A run spec reads it with no options; the package depends on `altar[gcs]`, and the `hf` mirror needs `altar[hf]`:
|
|
19
|
+
|
|
20
|
+
```toml
|
|
21
|
+
[sources.phylop]
|
|
22
|
+
type = "phylop100way"
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Hugging Face limits anonymous requests, so a large cohort read with `location = "hf"` can stall; read the bucket, or
|
|
26
|
+
pull a copy and name it with `dataset` ([published locations](https://kundajelab.github.io/altar/guides/published-datasets/#published-locations)).
|
|
27
|
+
|
|
28
|
+
In Python:
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
import asyncio
|
|
32
|
+
|
|
33
|
+
from altar_conservation import PhyloP100wayAnnotationSource
|
|
34
|
+
|
|
35
|
+
source = PhyloP100wayAnnotationSource.from_options() # the published copy
|
|
36
|
+
rows = asyncio.run(source.annotate(["chr7:140753336:A:T"]))
|
|
37
|
+
rows["chr7:140753336:A:T"]["phylop100way"] # 9.236
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The contract is `org.kundajelab.altar.annotation.phylop100way` (version 1.0.0, `PHYLOP100WAY_ANNOTATIONS`), one
|
|
41
|
+
float column `phylop100way`. The copy holds 2,837,915,604 scored bases of chr1–22, X, Y and M; a variant at an
|
|
42
|
+
unscored base, or on any other contig (alternate, unlocalized or unplaced), has no row.
|
|
43
|
+
|
|
44
|
+
`altar pull phylop100way --to <dir>` copies the published dataset (5.0 GB), checking each file's release; name the
|
|
45
|
+
copy with `dataset`. The options:
|
|
46
|
+
|
|
47
|
+
| Option | Meaning |
|
|
48
|
+
| --- | --- |
|
|
49
|
+
| `dataset` | An owned copy, as a path or a URI: a pulled copy, or one built with the builder below. |
|
|
50
|
+
| `location` | `gcs` (default) or `hf`, for the published copy. |
|
|
51
|
+
| `connection`, `storage_options` | How to reach a bucket. |
|
|
52
|
+
| `indels` | `skip` (default) scores SNVs only; `max` scores every variant (below). |
|
|
53
|
+
| `name` | Rename the source. |
|
|
54
|
+
| `genome_build` | Optional; must be `hg38`. |
|
|
55
|
+
| `prioritize_predicate` | In code only: a predicate over `phylop100way` that gates `prioritized`. |
|
|
56
|
+
|
|
57
|
+
`table` is not read yet: BigQuery would scan the 2.8-billion-row table for every chunk of variants, so the source
|
|
58
|
+
refuses it, before any other option, and asks for a pulled copy. An unknown option or `location` raises a
|
|
59
|
+
`ValueError` that lists the valid ones.
|
|
60
|
+
|
|
61
|
+
The track is on hg38 only. `altar run` stops before ingesting anything when the spec's `[run] genome_build` is
|
|
62
|
+
another assembly, naming the source.
|
|
63
|
+
|
|
64
|
+
With `indels = "max"`, a variant that is not an SNV is scored by the greatest score over the bases it touches, as
|
|
65
|
+
VEP takes them: a deletion's deleted bases (after the padding base), the two bases an insertion falls between, or
|
|
66
|
+
any other change's REF span. Unscored bases are ignored; a variant with none scored has no row. SNVs score the same
|
|
67
|
+
either way. The identity is the contract with release `2015-05` and `hg38`; with `indels = "max"` the release is
|
|
68
|
+
`2015-05+indels-max`, since it scores variants the default leaves out.
|
|
69
|
+
|
|
70
|
+
## Build the dataset
|
|
71
|
+
|
|
72
|
+
```console
|
|
73
|
+
pip install "altar-conservation[build]"
|
|
74
|
+
curl -O https://hgdownload.soe.ucsc.edu/goldenPath/hg38/phyloP100way/hg38.phyloP100way.bw
|
|
75
|
+
altar-conservation-build --track phylop100way --input hg38.phyloP100way.bw --output /scratch/phylop100way-hg38
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The `build` extra installs [pybigtools](https://github.com/jackh726/bigtools), which has wheels for Linux and
|
|
79
|
+
macOS. The builder checks the file's MD5 against UCSC's `md5sum.txt`, then converts each primary contig in its own
|
|
80
|
+
process (`--workers`, default one per contig up to the CPU count): it reads the contig's records, turns UCSC's 0-based
|
|
81
|
+
intervals into one row per base with a 1-based `pos`, checks that positions strictly increase, and writes
|
|
82
|
+
`chr=<contig>/scores.parquet` with `PartitionedWriter` in 128,000-row groups, `pos` delta-encoded, with the track,
|
|
83
|
+
release and genome build in every footer. Other contigs are skipped. On a 64-core server the build took 72 s
|
|
84
|
+
(15 s of it the MD5) and about 14 GB of memory across 25 processes, about 0.7 GB each. The output is a local
|
|
85
|
+
directory; copy it to a bucket yourself.
|
|
86
|
+
|
|
87
|
+
## Data terms and citation
|
|
88
|
+
|
|
89
|
+
UCSC's phyloP files are "freely available for public use"; UCSC asks you to credit the data and cite the Genome
|
|
90
|
+
Browser. Please cite:
|
|
91
|
+
|
|
92
|
+
- Pollard KS, Hubisz MJ, Rosenbloom KR, Siepel A. Detection of nonneutral substitution rates on mammalian phylogenies. *Genome
|
|
93
|
+
Research* 2010;20(1):110–121. doi:10.1101/gr.097857.109.
|
|
94
|
+
- Siepel A, et al. Evolutionarily conserved elements in vertebrate, insect, worm, and yeast genomes. *Genome
|
|
95
|
+
Research* 2005;15(8):1034–1050. doi:10.1101/gr.3715005.
|
|
96
|
+
- The UCSC Genome Browser's latest database paper (<https://genome.ucsc.edu/cite.html>): Casper J, et al.
|
|
97
|
+
*Nucleic Acids Research* 2026. PMID 41251146.
|
|
98
|
+
|
|
99
|
+
[Cite these data](https://kundajelab.github.io/altar/integrations/evidence/#cite-these-data) lists these with the
|
|
100
|
+
other datasets Altar publishes.
|
|
101
|
+
|
|
102
|
+
The binding ships no data. Its contract is its own; the source and layout come from `altar`, and the binding is
|
|
103
|
+
tested against Altar's conformance suites.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 2026 Riya Sinha
|
|
2
|
+
"""Base-wise conservation score tracks, such as UCSC phyloP 100-way, as Altar annotation sources."""
|
|
3
|
+
|
|
4
|
+
from altar_conservation.source import ConservationAnnotationSource, PhyloP100wayAnnotationSource
|
|
5
|
+
from altar_conservation.tracks import PHYLOP100WAY, PHYLOP100WAY_ANNOTATIONS, TRACKS, ConservationTrack
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"PHYLOP100WAY",
|
|
10
|
+
"PHYLOP100WAY_ANNOTATIONS",
|
|
11
|
+
"TRACKS",
|
|
12
|
+
"ConservationAnnotationSource",
|
|
13
|
+
"ConservationTrack",
|
|
14
|
+
"PhyloP100wayAnnotationSource",
|
|
15
|
+
]
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
# Copyright 2026 Riya Sinha
|
|
2
|
+
"""Build a track's position score dataset from its upstream bigWig file, such as UCSC's ``hg38.phyloP100way.bw``.
|
|
3
|
+
|
|
4
|
+
`build_dataset` checks the file's MD5 against the track's `upstream_md5`, then converts each primary contig
|
|
5
|
+
(chr1-22, X, Y and M) in its own spawned process: the bigWig's records are read with pybigtools, their 0-based,
|
|
6
|
+
half-open intervals become one row per base with a 1-based `pos` (`interval_rows`), and the rows are written with
|
|
7
|
+
`altar.sources.PartitionedWriter` in the layout `altar.sources.POSITION_SCORE_METADATA` describes: Hive
|
|
8
|
+
``chr=<contig>/``, 128,000-row groups, `pos` delta-encoded, the track, release and genome build in every footer.
|
|
9
|
+
Records on other contigs (alts, unplaced scaffolds) are not kept. The output is a local directory; copy it to a
|
|
10
|
+
bucket yourself.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
import argparse
|
|
15
|
+
import hashlib
|
|
16
|
+
import itertools
|
|
17
|
+
import multiprocessing
|
|
18
|
+
import os
|
|
19
|
+
import time
|
|
20
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import TYPE_CHECKING, Any
|
|
24
|
+
|
|
25
|
+
import numpy as np
|
|
26
|
+
import pyarrow as pa
|
|
27
|
+
import pyarrow.parquet as pq
|
|
28
|
+
from altar.sources import (
|
|
29
|
+
POSITION_SCORE_METADATA,
|
|
30
|
+
PartitionedWriter,
|
|
31
|
+
build_output,
|
|
32
|
+
position_score_release_fields,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
from altar_conservation.tracks import TRACKS, ConservationTrack
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
if TYPE_CHECKING:
|
|
39
|
+
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
PRIMARY_CONTIGS = (*(f"chr{number}" for number in range(1, 23)), "chrX", "chrY", "chrM")
|
|
43
|
+
"""The contigs a position score dataset keeps, in UCSC's spelling, which is also Altar's."""
|
|
44
|
+
SCHEMA = pa.schema(
|
|
45
|
+
[
|
|
46
|
+
pa.field("chr", pa.string(), nullable=False),
|
|
47
|
+
pa.field("pos", pa.int32(), nullable=False),
|
|
48
|
+
pa.field("score", pa.float32(), nullable=False),
|
|
49
|
+
]
|
|
50
|
+
)
|
|
51
|
+
"""One row per scored base: the contig, the 1-based position and the score."""
|
|
52
|
+
ROW_GROUP_ROWS = 128_000
|
|
53
|
+
PARQUET_OPTIONS: dict[str, Any] = {
|
|
54
|
+
# `chr` repeats and scores take few distinct values; positions increase, so they are delta-encoded, which needs
|
|
55
|
+
# dictionary encoding off for `pos`.
|
|
56
|
+
"use_dictionary": ["chr", "score"],
|
|
57
|
+
"column_encoding": {"pos": "DELTA_BINARY_PACKED"},
|
|
58
|
+
}
|
|
59
|
+
_STEM = "scores"
|
|
60
|
+
_LABEL = "position score"
|
|
61
|
+
# How many bigWig records a worker converts at once.
|
|
62
|
+
_CHUNK_RECORDS = 4_000_000
|
|
63
|
+
_RECORD = np.dtype([("start", np.int64), ("end", np.int64), ("value", np.float64)])
|
|
64
|
+
_MD5_BLOCK_BYTES = 8 * 1024 * 1024
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class ContigSummary:
|
|
69
|
+
"""What one contig contributed: its rows, its file and the file's size and row groups."""
|
|
70
|
+
|
|
71
|
+
contig: str
|
|
72
|
+
rows: int
|
|
73
|
+
file: Path
|
|
74
|
+
bytes: int
|
|
75
|
+
row_groups: int
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class BuildSummary:
|
|
80
|
+
"""What `build_dataset` wrote."""
|
|
81
|
+
|
|
82
|
+
track: str
|
|
83
|
+
contigs: tuple[ContigSummary, ...]
|
|
84
|
+
skipped_contigs: int
|
|
85
|
+
seconds: float
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def rows(self) -> int:
|
|
89
|
+
"""Every row written."""
|
|
90
|
+
return sum(contig.rows for contig in self.contigs)
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def files(self) -> tuple[Path, ...]:
|
|
94
|
+
"""The files written, one per contig."""
|
|
95
|
+
return tuple(contig.file for contig in self.contigs)
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def bytes(self) -> int:
|
|
99
|
+
"""The bytes written."""
|
|
100
|
+
return sum(contig.bytes for contig in self.contigs)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def row_groups(self) -> int:
|
|
104
|
+
"""The row groups written."""
|
|
105
|
+
return sum(contig.row_groups for contig in self.contigs)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def interval_rows(
|
|
109
|
+
starts: np.ndarray, ends: np.ndarray, values: np.ndarray, *, after: int = 0, label: str = "the input"
|
|
110
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
111
|
+
"""Return the rows of bigWig records: each base's 1-based position (int32) and its score (float32).
|
|
112
|
+
|
|
113
|
+
A record is a 0-based, half-open interval `[start, end)` with one value; a record of several bases becomes one
|
|
114
|
+
row per base, and unscored bases between records have no row. The records must be sorted and must not overlap,
|
|
115
|
+
so positions strictly increase, and must start after the 1-based position `after`, the last row of the previous
|
|
116
|
+
chunk of the same contig. Raises `ValueError` naming `label` otherwise, or for a value that is not finite.
|
|
117
|
+
"""
|
|
118
|
+
starts = np.asarray(starts, dtype=np.int64)
|
|
119
|
+
ends = np.asarray(ends, dtype=np.int64)
|
|
120
|
+
lengths = ends - starts
|
|
121
|
+
if (lengths < 1).any() or (starts < 0).any():
|
|
122
|
+
index = int(np.flatnonzero((lengths < 1) | (starts < 0))[0])
|
|
123
|
+
msg = f"{label} has an empty or negative interval [{starts[index]}, {ends[index]})"
|
|
124
|
+
raise ValueError(msg)
|
|
125
|
+
scores = np.asarray(values, dtype=np.float32)
|
|
126
|
+
if not np.isfinite(scores).all():
|
|
127
|
+
index = int(np.flatnonzero(~np.isfinite(scores))[0])
|
|
128
|
+
msg = f"{label} has a value that is not finite ({values[index]}) at [{starts[index]}, {ends[index]})"
|
|
129
|
+
raise ValueError(msg)
|
|
130
|
+
if len(starts) and int(lengths.max()) == 1:
|
|
131
|
+
positions = starts + 1 # one base per record, as UCSC's phyloP files store
|
|
132
|
+
else:
|
|
133
|
+
# Each base's offset from its record's first base, added to that base's 1-based position.
|
|
134
|
+
firsts = np.repeat(starts + 1, lengths)
|
|
135
|
+
offsets = np.arange(len(firsts), dtype=np.int64) - np.repeat(np.cumsum(lengths) - lengths, lengths)
|
|
136
|
+
positions = firsts + offsets
|
|
137
|
+
scores = np.repeat(scores, lengths)
|
|
138
|
+
if len(positions):
|
|
139
|
+
steps = np.diff(positions, prepend=after)
|
|
140
|
+
if (steps < 1).any():
|
|
141
|
+
index = int(np.flatnonzero(steps < 1)[0])
|
|
142
|
+
previous = after if index == 0 else int(positions[index - 1])
|
|
143
|
+
msg = (
|
|
144
|
+
f"{label} is not sorted, or has overlapping records: position {int(positions[index])} follows "
|
|
145
|
+
f"{previous}"
|
|
146
|
+
)
|
|
147
|
+
raise ValueError(msg)
|
|
148
|
+
if int(positions[-1]) > np.iinfo(np.int32).max:
|
|
149
|
+
msg = f"{label} has a position beyond int32: {int(positions[-1])}"
|
|
150
|
+
raise ValueError(msg)
|
|
151
|
+
return positions.astype(np.int32), scores
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def contig_batches(
|
|
155
|
+
contig: str, records: Iterable[tuple[int, int, float]], *, chunk_records: int = _CHUNK_RECORDS
|
|
156
|
+
) -> Iterator[pa.Table]:
|
|
157
|
+
"""Yield `contig`'s rows from its bigWig records, `(start, end, value)` in order, as tables of `SCHEMA`."""
|
|
158
|
+
iterator = iter(records)
|
|
159
|
+
after = 0
|
|
160
|
+
label = f"the bigWig's {contig}"
|
|
161
|
+
while True:
|
|
162
|
+
chunk = np.fromiter(itertools.islice(iterator, chunk_records), dtype=_RECORD)
|
|
163
|
+
if not len(chunk):
|
|
164
|
+
return
|
|
165
|
+
positions, scores = interval_rows(chunk["start"], chunk["end"], chunk["value"], after=after, label=label)
|
|
166
|
+
after = int(positions[-1])
|
|
167
|
+
yield pa.table(
|
|
168
|
+
[pa.repeat(pa.scalar(contig), len(positions)), pa.array(positions), pa.array(scores)], schema=SCHEMA
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def check_md5(path: str | os.PathLike[str], expected: str) -> None:
|
|
173
|
+
"""Raise `ValueError` unless the file's MD5 is `expected`, reading it in blocks."""
|
|
174
|
+
digest = hashlib.md5(usedforsecurity=False)
|
|
175
|
+
with Path(path).open("rb") as handle:
|
|
176
|
+
while block := handle.read(_MD5_BLOCK_BYTES):
|
|
177
|
+
digest.update(block)
|
|
178
|
+
if digest.hexdigest() != expected.lower():
|
|
179
|
+
msg = f"{path} has MD5 {digest.hexdigest()}, not {expected}, the checksum its host publishes; download it again"
|
|
180
|
+
raise ValueError(msg)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def build_dataset(
|
|
184
|
+
track: ConservationTrack,
|
|
185
|
+
bigwig: str | os.PathLike[str],
|
|
186
|
+
output: str | os.PathLike[str],
|
|
187
|
+
*,
|
|
188
|
+
overwrite: bool = False,
|
|
189
|
+
workers: int | None = None,
|
|
190
|
+
) -> BuildSummary:
|
|
191
|
+
"""Build `track`'s position score dataset from its upstream bigWig, a local file, in the local directory `output`.
|
|
192
|
+
|
|
193
|
+
The file's MD5 must be the track's `upstream_md5`. Each primary contig the file holds is converted in its own
|
|
194
|
+
spawned process, `workers` at a time (default: one per contig, up to the CPU count); other contigs are skipped.
|
|
195
|
+
`output` must be empty or absent; with `overwrite`, a previous build there is replaced. A failed build leaves
|
|
196
|
+
nothing behind.
|
|
197
|
+
"""
|
|
198
|
+
started = time.monotonic()
|
|
199
|
+
source = Path(bigwig)
|
|
200
|
+
if not source.is_file():
|
|
201
|
+
msg = f"no bigWig file at {source}"
|
|
202
|
+
raise FileNotFoundError(msg)
|
|
203
|
+
check_md5(source, track.upstream_md5)
|
|
204
|
+
contigs = _contigs(source)
|
|
205
|
+
primary = [contig for contig in PRIMARY_CONTIGS if contig in contigs]
|
|
206
|
+
if not primary:
|
|
207
|
+
msg = f"{source} holds none of the primary contigs {', '.join(PRIMARY_CONTIGS)}"
|
|
208
|
+
raise ValueError(msg)
|
|
209
|
+
if workers is not None and workers < 1:
|
|
210
|
+
msg = f"workers must be positive, got {workers}"
|
|
211
|
+
raise ValueError(msg)
|
|
212
|
+
count = min(len(primary), workers or os.cpu_count() or 1)
|
|
213
|
+
metadata = track_metadata(track)
|
|
214
|
+
with build_output(Path(output), overwrite=overwrite, partition_column="chr", label=_LABEL) as directory:
|
|
215
|
+
tasks = [(source, directory, contig, metadata) for contig in primary]
|
|
216
|
+
summaries = _convert(tasks, count)
|
|
217
|
+
return BuildSummary(
|
|
218
|
+
track=track.name,
|
|
219
|
+
contigs=tuple(summary for summary in summaries if summary is not None),
|
|
220
|
+
skipped_contigs=len(contigs) - len(primary),
|
|
221
|
+
seconds=time.monotonic() - started,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def track_metadata(track: ConservationTrack) -> dict[str, str]:
|
|
226
|
+
"""Return the footer metadata every file of `track`'s dataset records (`POSITION_SCORE_METADATA`)."""
|
|
227
|
+
return POSITION_SCORE_METADATA.write(
|
|
228
|
+
position_score_release_fields(contract=track.contract, release=track.release, genome_build=track.genome_build)
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _contigs(source: Path) -> dict[str, int]:
|
|
233
|
+
import pybigtools # noqa: PLC0415 - the `build` extra
|
|
234
|
+
|
|
235
|
+
with pybigtools.open(str(source)) as reader:
|
|
236
|
+
return dict(reader.chroms())
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _convert(tasks: Sequence[tuple[Path, Path, str, Mapping[str, str]]], count: int) -> list[ContigSummary | None]:
|
|
240
|
+
if count == 1:
|
|
241
|
+
return [_convert_contig(*task) for task in tasks]
|
|
242
|
+
# `spawn` starts each worker from a fresh interpreter: forking a process with threads running can deadlock it.
|
|
243
|
+
context = multiprocessing.get_context("spawn")
|
|
244
|
+
with ProcessPoolExecutor(max_workers=count, mp_context=context) as pool:
|
|
245
|
+
futures = [pool.submit(_convert_contig, *task) for task in tasks]
|
|
246
|
+
try:
|
|
247
|
+
return [future.result() for future in futures]
|
|
248
|
+
except BaseException:
|
|
249
|
+
for future in futures:
|
|
250
|
+
future.cancel()
|
|
251
|
+
raise
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _convert_contig(source: Path, output: Path, contig: str, metadata: Mapping[str, str]) -> ContigSummary | None:
|
|
255
|
+
import pybigtools # noqa: PLC0415 - the `build` extra
|
|
256
|
+
|
|
257
|
+
with pybigtools.open(str(source)) as reader:
|
|
258
|
+
return write_contig(output, contig, reader.records(contig), metadata)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def write_contig(
|
|
262
|
+
output: Path, contig: str, records: Iterable[tuple[int, int, float]], metadata: Mapping[str, str]
|
|
263
|
+
) -> ContigSummary | None:
|
|
264
|
+
"""Write `contig`'s rows from its bigWig records to `output/chr=<contig>/`; return `None` if it has none.
|
|
265
|
+
|
|
266
|
+
`records` are `(start, end, value)`, 0-based and half-open, in order, as pybigtools reads them; `metadata` is
|
|
267
|
+
`track_metadata`'s. The file is written under a ``.partial`` name and renamed when it is complete.
|
|
268
|
+
"""
|
|
269
|
+
writer = PartitionedWriter(
|
|
270
|
+
output,
|
|
271
|
+
_STEM,
|
|
272
|
+
schema=SCHEMA,
|
|
273
|
+
partition_column="chr",
|
|
274
|
+
row_group_rows=ROW_GROUP_ROWS,
|
|
275
|
+
metadata=metadata,
|
|
276
|
+
parquet_options=PARQUET_OPTIONS,
|
|
277
|
+
)
|
|
278
|
+
rows = 0
|
|
279
|
+
try:
|
|
280
|
+
for batch in contig_batches(contig, records):
|
|
281
|
+
writer.append_batch(contig, batch)
|
|
282
|
+
rows += batch.num_rows
|
|
283
|
+
files = writer.commit()
|
|
284
|
+
except BaseException:
|
|
285
|
+
writer.abort()
|
|
286
|
+
raise
|
|
287
|
+
if not files:
|
|
288
|
+
return None
|
|
289
|
+
[file] = files
|
|
290
|
+
return ContigSummary(
|
|
291
|
+
contig=contig,
|
|
292
|
+
rows=rows,
|
|
293
|
+
file=file,
|
|
294
|
+
bytes=file.stat().st_size,
|
|
295
|
+
row_groups=pq.ParquetFile(file).metadata.num_row_groups,
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def main() -> None:
|
|
300
|
+
"""Run the conservation track builder command."""
|
|
301
|
+
parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0])
|
|
302
|
+
parser.add_argument("--track", required=True, choices=sorted(TRACKS), help="The track to build")
|
|
303
|
+
parser.add_argument("--input", required=True, help="The track's upstream bigWig file, downloaded locally")
|
|
304
|
+
parser.add_argument("--output", required=True, help="The local directory to write the dataset to")
|
|
305
|
+
parser.add_argument("--overwrite", action="store_true", help="Replace a previous build in --output")
|
|
306
|
+
parser.add_argument("--workers", type=int, help="Contigs to convert at once, in processes (default: CPU count)")
|
|
307
|
+
args = parser.parse_args()
|
|
308
|
+
summary = build_dataset(TRACKS[args.track], args.input, args.output, overwrite=args.overwrite, workers=args.workers)
|
|
309
|
+
for contig in summary.contigs:
|
|
310
|
+
print(f"{contig.contig}\t{contig.rows}") # noqa: T201 - command-line report
|
|
311
|
+
print( # noqa: T201
|
|
312
|
+
f"wrote {summary.rows} rows of {summary.track} in {len(summary.files)} files and {summary.row_groups} row "
|
|
313
|
+
f"groups ({summary.bytes / 1e9:.2f} GB) to {args.output}; skipped {summary.skipped_contigs} non-primary "
|
|
314
|
+
f"contigs; {summary.seconds:.0f} s"
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
if __name__ == "__main__":
|
|
319
|
+
main()
|
|
File without changes
|