altar-conservation 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,33 @@
1
+ *.pyc
2
+ *~
3
+ **/__pycache__/*
4
+ *.swp
5
+ .vscode/
6
+ .idea/
7
+ .DS_Store
8
+ .env*
9
+ .mypy_cache/
10
+ .ruff_cache/
11
+ .pytest_cache/
12
+ .coverage
13
+ htmlcov/
14
+ coverage.xml
15
+ pytest.xml
16
+ .hypothesis/
17
+ .python-version
18
+ .venv
19
+ .venv-*/
20
+ *.egg-info/
21
+ build/
22
+ dist/
23
+ site/
24
+ *.sqlite3
25
+ *.log
26
+
27
+ # Generated variant indexes and datasets. The small canonical gene table is tracked.
28
+ altar/altar/variants/data/ccres.dnatree
29
+ altar/altar/variants/data/region_annotations.parquet
30
+ altar/altar/variants/data/variants.pkl.gz
31
+ altar/altar/variants/data/region_provenance.json
32
+ altar/altar/variants/data/raw/
33
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Riya Sinha
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,128 @@
1
+ Metadata-Version: 2.5
2
+ Name: altar-conservation
3
+ Version: 0.3.0
4
+ Summary: Base-wise conservation score tracks (UCSC phyloP) for Altar
5
+ Project-URL: Documentation, https://kundajelab.github.io/altar/
6
+ Author: Riya Sinha
7
+ License-Expression: MIT
8
+ License-File: LICENSE
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Programming Language :: Python :: 3.13
15
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
16
+ Classifier: Typing :: Typed
17
+ Requires-Python: >=3.12
18
+ Requires-Dist: altar[gcs]<0.4,>=0.3.0
19
+ Provides-Extra: build
20
+ Requires-Dist: pybigtools>=0.3.0; extra == 'build'
21
+ Provides-Extra: test
22
+ Requires-Dist: pytest-asyncio>=0.24; extra == 'test'
23
+ Requires-Dist: pytest>=8; extra == 'test'
24
+ Description-Content-Type: text/markdown
25
+
26
+ # altar-conservation
27
+
28
+ This binding adds base-wise conservation scores to variants. Its one track today is UCSC's hg38 phyloP 100-way
29
+ (`hg38.phyloP100way.bw`, May 2015): `phylop100way`, an annotation source (`PhyloP100wayAnnotationSource`), adds a
30
+ `phylop100way` column, the phyloP score at the variant's base, from a Parquet copy of the track: the one the
31
+ Kundaje lab publishes, or one you own. phyloP is a signed -log10 p-value: positive where a base evolves more slowly
32
+ than neutral across 100 vertebrates (conserved), negative where it evolves faster.
33
+
34
+ ## Annotate variants from the published copy
35
+
36
+ The Kundaje lab publishes the track in Altar's position score layout (`PHYLOP100WAY.published`):
37
+
38
+ | Location | URI |
39
+ | --- | --- |
40
+ | `gcs` (default) | `gs://kundajelab-altar/conservation/phylop100way-hg38` |
41
+ | `hf` | `hf://datasets/altar-genomics/phylop100way-hg38` ([dataset card](https://huggingface.co/datasets/altar-genomics/phylop100way-hg38)) |
42
+
43
+ A run spec reads it with no options; the package depends on `altar[gcs]`, and the `hf` mirror needs `altar[hf]`:
44
+
45
+ ```toml
46
+ [sources.phylop]
47
+ type = "phylop100way"
48
+ ```
49
+
50
+ Hugging Face limits anonymous requests, so a large cohort read with `location = "hf"` can stall; read the bucket, or
51
+ pull a copy and name it with `dataset` ([published locations](https://kundajelab.github.io/altar/guides/published-datasets/#published-locations)).
52
+
53
+ In Python:
54
+
55
+ ```python
56
+ import asyncio
57
+
58
+ from altar_conservation import PhyloP100wayAnnotationSource
59
+
60
+ source = PhyloP100wayAnnotationSource.from_options() # the published copy
61
+ rows = asyncio.run(source.annotate(["chr7:140753336:A:T"]))
62
+ rows["chr7:140753336:A:T"]["phylop100way"] # 9.236
63
+ ```
64
+
65
+ The contract is `org.kundajelab.altar.annotation.phylop100way` (version 1.0.0, `PHYLOP100WAY_ANNOTATIONS`), one
66
+ float column `phylop100way`. The copy holds 2,837,915,604 scored bases of chr1–22, X, Y and M; a variant at an
67
+ unscored base, or on any other contig (alternate, unlocalized or unplaced), has no row.
68
+
69
+ `altar pull phylop100way --to <dir>` copies the published dataset (5.0 GB), checking each file's release; name the
70
+ copy with `dataset`. The options:
71
+
72
+ | Option | Meaning |
73
+ | --- | --- |
74
+ | `dataset` | An owned copy, as a path or a URI: a pulled copy, or one built with the builder below. |
75
+ | `location` | `gcs` (default) or `hf`, for the published copy. |
76
+ | `connection`, `storage_options` | How to reach a bucket. |
77
+ | `indels` | `skip` (default) scores SNVs only; `max` scores every variant (below). |
78
+ | `name` | Rename the source. |
79
+ | `genome_build` | Optional; must be `hg38`. |
80
+ | `prioritize_predicate` | In code only: a predicate over `phylop100way` that gates `prioritized`. |
81
+
82
+ `table` is not read yet: BigQuery would scan the 2.8-billion-row table for every chunk of variants, so the source
83
+ refuses it, before any other option, and asks for a pulled copy. An unknown option or `location` raises a
84
+ `ValueError` that lists the valid ones.
85
+
86
+ The track is on hg38 only. `altar run` stops before ingesting anything when the spec's `[run] genome_build` is
87
+ another assembly, naming the source.
88
+
89
+ With `indels = "max"`, a variant that is not an SNV is scored by the greatest score over the bases it touches, as
90
+ VEP takes them: a deletion's deleted bases (after the padding base), the two bases an insertion falls between, or
91
+ any other change's REF span. Unscored bases are ignored; a variant with none scored has no row. SNVs score the same
92
+ either way. The identity is the contract with release `2015-05` and `hg38`; with `indels = "max"` the release is
93
+ `2015-05+indels-max`, since it scores variants the default leaves out.
94
+
95
+ ## Build the dataset
96
+
97
+ ```console
98
+ pip install "altar-conservation[build]"
99
+ curl -O https://hgdownload.soe.ucsc.edu/goldenPath/hg38/phyloP100way/hg38.phyloP100way.bw
100
+ altar-conservation-build --track phylop100way --input hg38.phyloP100way.bw --output /scratch/phylop100way-hg38
101
+ ```
102
+
103
+ The `build` extra installs [pybigtools](https://github.com/jackh726/bigtools), which has wheels for Linux and
104
+ macOS. The builder checks the file's MD5 against UCSC's `md5sum.txt`, then converts each primary contig in its own
105
+ process (`--workers`, default one per contig up to the CPU count): it reads the contig's records, turns UCSC's 0-based
106
+ intervals into one row per base with a 1-based `pos`, checks that positions strictly increase, and writes
107
+ `chr=<contig>/scores.parquet` with `PartitionedWriter` in 128,000-row groups, `pos` delta-encoded, with the track,
108
+ release and genome build in every footer. Other contigs are skipped. On a 64-core server the build took 72 s
109
+ (15 s of it the MD5) and about 14 GB of memory across 25 processes, about 0.7 GB each. The output is a local
110
+ directory; copy it to a bucket yourself.
111
+
112
+ ## Data terms and citation
113
+
114
+ UCSC's phyloP files are "freely available for public use"; UCSC asks you to credit the data and cite the Genome
115
+ Browser. Please cite:
116
+
117
+ - Pollard KS, Hubisz MJ, Rosenbloom KR, Siepel A. Detection of nonneutral substitution rates on mammalian phylogenies. *Genome
118
+ Research* 2010;20(1):110–121. doi:10.1101/gr.097857.109.
119
+ - Siepel A, et al. Evolutionarily conserved elements in vertebrate, insect, worm, and yeast genomes. *Genome
120
+ Research* 2005;15(8):1034–1050. doi:10.1101/gr.3715005.
121
+ - The UCSC Genome Browser's latest database paper (<https://genome.ucsc.edu/cite.html>): Casper J, et al.
122
+ *Nucleic Acids Research* 2026. PMID 41251146.
123
+
124
+ [Cite these data](https://kundajelab.github.io/altar/integrations/evidence/#cite-these-data) lists these with the
125
+ other datasets Altar publishes.
126
+
127
+ The binding ships no data. Its contract is its own; the source and layout come from `altar`, and the binding is
128
+ tested against Altar's conformance suites.
@@ -0,0 +1,103 @@
1
+ # altar-conservation
2
+
3
+ This binding adds base-wise conservation scores to variants. Its one track today is UCSC's hg38 phyloP 100-way
4
+ (`hg38.phyloP100way.bw`, May 2015): `phylop100way`, an annotation source (`PhyloP100wayAnnotationSource`), adds a
5
+ `phylop100way` column, the phyloP score at the variant's base, from a Parquet copy of the track: the one the
6
+ Kundaje lab publishes, or one you own. phyloP is a signed -log10 p-value: positive where a base evolves more slowly
7
+ than neutral across 100 vertebrates (conserved), negative where it evolves faster.
8
+
9
+ ## Annotate variants from the published copy
10
+
11
+ The Kundaje lab publishes the track in Altar's position score layout (`PHYLOP100WAY.published`):
12
+
13
+ | Location | URI |
14
+ | --- | --- |
15
+ | `gcs` (default) | `gs://kundajelab-altar/conservation/phylop100way-hg38` |
16
+ | `hf` | `hf://datasets/altar-genomics/phylop100way-hg38` ([dataset card](https://huggingface.co/datasets/altar-genomics/phylop100way-hg38)) |
17
+
18
+ A run spec reads it with no options; the package depends on `altar[gcs]`, and the `hf` mirror needs `altar[hf]`:
19
+
20
+ ```toml
21
+ [sources.phylop]
22
+ type = "phylop100way"
23
+ ```
24
+
25
+ Hugging Face limits anonymous requests, so a large cohort read with `location = "hf"` can stall; read the bucket, or
26
+ pull a copy and name it with `dataset` ([published locations](https://kundajelab.github.io/altar/guides/published-datasets/#published-locations)).
27
+
28
+ In Python:
29
+
30
+ ```python
31
+ import asyncio
32
+
33
+ from altar_conservation import PhyloP100wayAnnotationSource
34
+
35
+ source = PhyloP100wayAnnotationSource.from_options() # the published copy
36
+ rows = asyncio.run(source.annotate(["chr7:140753336:A:T"]))
37
+ rows["chr7:140753336:A:T"]["phylop100way"] # 9.236
38
+ ```
39
+
40
+ The contract is `org.kundajelab.altar.annotation.phylop100way` (version 1.0.0, `PHYLOP100WAY_ANNOTATIONS`), one
41
+ float column `phylop100way`. The copy holds 2,837,915,604 scored bases of chr1–22, X, Y and M; a variant at an
42
+ unscored base, or on any other contig (alternate, unlocalized or unplaced), has no row.
43
+
44
+ `altar pull phylop100way --to <dir>` copies the published dataset (5.0 GB), checking each file's release; name the
45
+ copy with `dataset`. The options:
46
+
47
+ | Option | Meaning |
48
+ | --- | --- |
49
+ | `dataset` | An owned copy, as a path or a URI: a pulled copy, or one built with the builder below. |
50
+ | `location` | `gcs` (default) or `hf`, for the published copy. |
51
+ | `connection`, `storage_options` | How to reach a bucket. |
52
+ | `indels` | `skip` (default) scores SNVs only; `max` scores every variant (below). |
53
+ | `name` | Rename the source. |
54
+ | `genome_build` | Optional; must be `hg38`. |
55
+ | `prioritize_predicate` | In code only: a predicate over `phylop100way` that gates `prioritized`. |
56
+
57
+ `table` is not read yet: BigQuery would scan the 2.8-billion-row table for every chunk of variants, so the source
58
+ refuses it, before any other option, and asks for a pulled copy. An unknown option or `location` raises a
59
+ `ValueError` that lists the valid ones.
60
+
61
+ The track is on hg38 only. `altar run` stops before ingesting anything when the spec's `[run] genome_build` is
62
+ another assembly, naming the source.
63
+
64
+ With `indels = "max"`, a variant that is not an SNV is scored by the greatest score over the bases it touches, as
65
+ VEP takes them: a deletion's deleted bases (after the padding base), the two bases an insertion falls between, or
66
+ any other change's REF span. Unscored bases are ignored; a variant with none scored has no row. SNVs score the same
67
+ either way. The identity is the contract with release `2015-05` and `hg38`; with `indels = "max"` the release is
68
+ `2015-05+indels-max`, since it scores variants the default leaves out.
69
+
70
+ ## Build the dataset
71
+
72
+ ```console
73
+ pip install "altar-conservation[build]"
74
+ curl -O https://hgdownload.soe.ucsc.edu/goldenPath/hg38/phyloP100way/hg38.phyloP100way.bw
75
+ altar-conservation-build --track phylop100way --input hg38.phyloP100way.bw --output /scratch/phylop100way-hg38
76
+ ```
77
+
78
+ The `build` extra installs [pybigtools](https://github.com/jackh726/bigtools), which has wheels for Linux and
79
+ macOS. The builder checks the file's MD5 against UCSC's `md5sum.txt`, then converts each primary contig in its own
80
+ process (`--workers`, default one per contig up to the CPU count): it reads the contig's records, turns UCSC's 0-based
81
+ intervals into one row per base with a 1-based `pos`, checks that positions strictly increase, and writes
82
+ `chr=<contig>/scores.parquet` with `PartitionedWriter` in 128,000-row groups, `pos` delta-encoded, with the track,
83
+ release and genome build in every footer. Other contigs are skipped. On a 64-core server the build took 72 s
84
+ (15 s of it the MD5) and about 14 GB of memory across 25 processes, about 0.7 GB each. The output is a local
85
+ directory; copy it to a bucket yourself.
86
+
87
+ ## Data terms and citation
88
+
89
+ UCSC's phyloP files are "freely available for public use"; UCSC asks you to credit the data and cite the Genome
90
+ Browser. Please cite:
91
+
92
+ - Pollard KS, Hubisz MJ, Rosenbloom KR, Siepel A. Detection of nonneutral substitution rates on mammalian phylogenies. *Genome
93
+ Research* 2010;20(1):110–121. doi:10.1101/gr.097857.109.
94
+ - Siepel A, et al. Evolutionarily conserved elements in vertebrate, insect, worm, and yeast genomes. *Genome
95
+ Research* 2005;15(8):1034–1050. doi:10.1101/gr.3715005.
96
+ - The UCSC Genome Browser's latest database paper (<https://genome.ucsc.edu/cite.html>): Casper J, et al.
97
+ *Nucleic Acids Research* 2026. PMID 41251146.
98
+
99
+ [Cite these data](https://kundajelab.github.io/altar/integrations/evidence/#cite-these-data) lists these with the
100
+ other datasets Altar publishes.
101
+
102
+ The binding ships no data. Its contract is its own; the source and layout come from `altar`, and the binding is
103
+ tested against Altar's conformance suites.
@@ -0,0 +1,15 @@
1
+ # Copyright 2026 Riya Sinha
2
+ """Base-wise conservation score tracks, such as UCSC phyloP 100-way, as Altar annotation sources."""
3
+
4
+ from altar_conservation.source import ConservationAnnotationSource, PhyloP100wayAnnotationSource
5
+ from altar_conservation.tracks import PHYLOP100WAY, PHYLOP100WAY_ANNOTATIONS, TRACKS, ConservationTrack
6
+
7
+
8
+ __all__ = [
9
+ "PHYLOP100WAY",
10
+ "PHYLOP100WAY_ANNOTATIONS",
11
+ "TRACKS",
12
+ "ConservationAnnotationSource",
13
+ "ConservationTrack",
14
+ "PhyloP100wayAnnotationSource",
15
+ ]
@@ -0,0 +1,319 @@
1
+ # Copyright 2026 Riya Sinha
2
+ """Build a track's position score dataset from its upstream bigWig file, such as UCSC's ``hg38.phyloP100way.bw``.
3
+
4
+ `build_dataset` checks the file's MD5 against the track's `upstream_md5`, then converts each primary contig
5
+ (chr1-22, X, Y and M) in its own spawned process: the bigWig's records are read with pybigtools, their 0-based,
6
+ half-open intervals become one row per base with a 1-based `pos` (`interval_rows`), and the rows are written with
7
+ `altar.sources.PartitionedWriter` in the layout `altar.sources.POSITION_SCORE_METADATA` describes: Hive
8
+ ``chr=<contig>/``, 128,000-row groups, `pos` delta-encoded, the track, release and genome build in every footer.
9
+ Records on other contigs (alts, unplaced scaffolds) are not kept. The output is a local directory; copy it to a
10
+ bucket yourself.
11
+ """
12
+
13
+ from __future__ import annotations
14
+ import argparse
15
+ import hashlib
16
+ import itertools
17
+ import multiprocessing
18
+ import os
19
+ import time
20
+ from concurrent.futures import ProcessPoolExecutor
21
+ from dataclasses import dataclass
22
+ from pathlib import Path
23
+ from typing import TYPE_CHECKING, Any
24
+
25
+ import numpy as np
26
+ import pyarrow as pa
27
+ import pyarrow.parquet as pq
28
+ from altar.sources import (
29
+ POSITION_SCORE_METADATA,
30
+ PartitionedWriter,
31
+ build_output,
32
+ position_score_release_fields,
33
+ )
34
+
35
+ from altar_conservation.tracks import TRACKS, ConservationTrack
36
+
37
+
38
+ if TYPE_CHECKING:
39
+ from collections.abc import Iterable, Iterator, Mapping, Sequence
40
+
41
+
42
+ PRIMARY_CONTIGS = (*(f"chr{number}" for number in range(1, 23)), "chrX", "chrY", "chrM")
43
+ """The contigs a position score dataset keeps, in UCSC's spelling, which is also Altar's."""
44
+ SCHEMA = pa.schema(
45
+ [
46
+ pa.field("chr", pa.string(), nullable=False),
47
+ pa.field("pos", pa.int32(), nullable=False),
48
+ pa.field("score", pa.float32(), nullable=False),
49
+ ]
50
+ )
51
+ """One row per scored base: the contig, the 1-based position and the score."""
52
+ ROW_GROUP_ROWS = 128_000
53
+ PARQUET_OPTIONS: dict[str, Any] = {
54
+ # `chr` repeats and scores take few distinct values; positions increase, so they are delta-encoded, which needs
55
+ # dictionary encoding off for `pos`.
56
+ "use_dictionary": ["chr", "score"],
57
+ "column_encoding": {"pos": "DELTA_BINARY_PACKED"},
58
+ }
59
+ _STEM = "scores"
60
+ _LABEL = "position score"
61
+ # How many bigWig records a worker converts at once.
62
+ _CHUNK_RECORDS = 4_000_000
63
+ _RECORD = np.dtype([("start", np.int64), ("end", np.int64), ("value", np.float64)])
64
+ _MD5_BLOCK_BYTES = 8 * 1024 * 1024
65
+
66
+
67
+ @dataclass(frozen=True)
68
+ class ContigSummary:
69
+ """What one contig contributed: its rows, its file and the file's size and row groups."""
70
+
71
+ contig: str
72
+ rows: int
73
+ file: Path
74
+ bytes: int
75
+ row_groups: int
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class BuildSummary:
80
+ """What `build_dataset` wrote."""
81
+
82
+ track: str
83
+ contigs: tuple[ContigSummary, ...]
84
+ skipped_contigs: int
85
+ seconds: float
86
+
87
+ @property
88
+ def rows(self) -> int:
89
+ """Every row written."""
90
+ return sum(contig.rows for contig in self.contigs)
91
+
92
+ @property
93
+ def files(self) -> tuple[Path, ...]:
94
+ """The files written, one per contig."""
95
+ return tuple(contig.file for contig in self.contigs)
96
+
97
+ @property
98
+ def bytes(self) -> int:
99
+ """The bytes written."""
100
+ return sum(contig.bytes for contig in self.contigs)
101
+
102
+ @property
103
+ def row_groups(self) -> int:
104
+ """The row groups written."""
105
+ return sum(contig.row_groups for contig in self.contigs)
106
+
107
+
108
+ def interval_rows(
109
+ starts: np.ndarray, ends: np.ndarray, values: np.ndarray, *, after: int = 0, label: str = "the input"
110
+ ) -> tuple[np.ndarray, np.ndarray]:
111
+ """Return the rows of bigWig records: each base's 1-based position (int32) and its score (float32).
112
+
113
+ A record is a 0-based, half-open interval `[start, end)` with one value; a record of several bases becomes one
114
+ row per base, and unscored bases between records have no row. The records must be sorted and must not overlap,
115
+ so positions strictly increase, and must start after the 1-based position `after`, the last row of the previous
116
+ chunk of the same contig. Raises `ValueError` naming `label` otherwise, or for a value that is not finite.
117
+ """
118
+ starts = np.asarray(starts, dtype=np.int64)
119
+ ends = np.asarray(ends, dtype=np.int64)
120
+ lengths = ends - starts
121
+ if (lengths < 1).any() or (starts < 0).any():
122
+ index = int(np.flatnonzero((lengths < 1) | (starts < 0))[0])
123
+ msg = f"{label} has an empty or negative interval [{starts[index]}, {ends[index]})"
124
+ raise ValueError(msg)
125
+ scores = np.asarray(values, dtype=np.float32)
126
+ if not np.isfinite(scores).all():
127
+ index = int(np.flatnonzero(~np.isfinite(scores))[0])
128
+ msg = f"{label} has a value that is not finite ({values[index]}) at [{starts[index]}, {ends[index]})"
129
+ raise ValueError(msg)
130
+ if len(starts) and int(lengths.max()) == 1:
131
+ positions = starts + 1 # one base per record, as UCSC's phyloP files store
132
+ else:
133
+ # Each base's offset from its record's first base, added to that base's 1-based position.
134
+ firsts = np.repeat(starts + 1, lengths)
135
+ offsets = np.arange(len(firsts), dtype=np.int64) - np.repeat(np.cumsum(lengths) - lengths, lengths)
136
+ positions = firsts + offsets
137
+ scores = np.repeat(scores, lengths)
138
+ if len(positions):
139
+ steps = np.diff(positions, prepend=after)
140
+ if (steps < 1).any():
141
+ index = int(np.flatnonzero(steps < 1)[0])
142
+ previous = after if index == 0 else int(positions[index - 1])
143
+ msg = (
144
+ f"{label} is not sorted, or has overlapping records: position {int(positions[index])} follows "
145
+ f"{previous}"
146
+ )
147
+ raise ValueError(msg)
148
+ if int(positions[-1]) > np.iinfo(np.int32).max:
149
+ msg = f"{label} has a position beyond int32: {int(positions[-1])}"
150
+ raise ValueError(msg)
151
+ return positions.astype(np.int32), scores
152
+
153
+
154
+ def contig_batches(
155
+ contig: str, records: Iterable[tuple[int, int, float]], *, chunk_records: int = _CHUNK_RECORDS
156
+ ) -> Iterator[pa.Table]:
157
+ """Yield `contig`'s rows from its bigWig records, `(start, end, value)` in order, as tables of `SCHEMA`."""
158
+ iterator = iter(records)
159
+ after = 0
160
+ label = f"the bigWig's {contig}"
161
+ while True:
162
+ chunk = np.fromiter(itertools.islice(iterator, chunk_records), dtype=_RECORD)
163
+ if not len(chunk):
164
+ return
165
+ positions, scores = interval_rows(chunk["start"], chunk["end"], chunk["value"], after=after, label=label)
166
+ after = int(positions[-1])
167
+ yield pa.table(
168
+ [pa.repeat(pa.scalar(contig), len(positions)), pa.array(positions), pa.array(scores)], schema=SCHEMA
169
+ )
170
+
171
+
172
+ def check_md5(path: str | os.PathLike[str], expected: str) -> None:
173
+ """Raise `ValueError` unless the file's MD5 is `expected`, reading it in blocks."""
174
+ digest = hashlib.md5(usedforsecurity=False)
175
+ with Path(path).open("rb") as handle:
176
+ while block := handle.read(_MD5_BLOCK_BYTES):
177
+ digest.update(block)
178
+ if digest.hexdigest() != expected.lower():
179
+ msg = f"{path} has MD5 {digest.hexdigest()}, not {expected}, the checksum its host publishes; download it again"
180
+ raise ValueError(msg)
181
+
182
+
183
+ def build_dataset(
184
+ track: ConservationTrack,
185
+ bigwig: str | os.PathLike[str],
186
+ output: str | os.PathLike[str],
187
+ *,
188
+ overwrite: bool = False,
189
+ workers: int | None = None,
190
+ ) -> BuildSummary:
191
+ """Build `track`'s position score dataset from its upstream bigWig, a local file, in the local directory `output`.
192
+
193
+ The file's MD5 must be the track's `upstream_md5`. Each primary contig the file holds is converted in its own
194
+ spawned process, `workers` at a time (default: one per contig, up to the CPU count); other contigs are skipped.
195
+ `output` must be empty or absent; with `overwrite`, a previous build there is replaced. A failed build leaves
196
+ nothing behind.
197
+ """
198
+ started = time.monotonic()
199
+ source = Path(bigwig)
200
+ if not source.is_file():
201
+ msg = f"no bigWig file at {source}"
202
+ raise FileNotFoundError(msg)
203
+ check_md5(source, track.upstream_md5)
204
+ contigs = _contigs(source)
205
+ primary = [contig for contig in PRIMARY_CONTIGS if contig in contigs]
206
+ if not primary:
207
+ msg = f"{source} holds none of the primary contigs {', '.join(PRIMARY_CONTIGS)}"
208
+ raise ValueError(msg)
209
+ if workers is not None and workers < 1:
210
+ msg = f"workers must be positive, got {workers}"
211
+ raise ValueError(msg)
212
+ count = min(len(primary), workers or os.cpu_count() or 1)
213
+ metadata = track_metadata(track)
214
+ with build_output(Path(output), overwrite=overwrite, partition_column="chr", label=_LABEL) as directory:
215
+ tasks = [(source, directory, contig, metadata) for contig in primary]
216
+ summaries = _convert(tasks, count)
217
+ return BuildSummary(
218
+ track=track.name,
219
+ contigs=tuple(summary for summary in summaries if summary is not None),
220
+ skipped_contigs=len(contigs) - len(primary),
221
+ seconds=time.monotonic() - started,
222
+ )
223
+
224
+
225
+ def track_metadata(track: ConservationTrack) -> dict[str, str]:
226
+ """Return the footer metadata every file of `track`'s dataset records (`POSITION_SCORE_METADATA`)."""
227
+ return POSITION_SCORE_METADATA.write(
228
+ position_score_release_fields(contract=track.contract, release=track.release, genome_build=track.genome_build)
229
+ )
230
+
231
+
232
+ def _contigs(source: Path) -> dict[str, int]:
233
+ import pybigtools # noqa: PLC0415 - the `build` extra
234
+
235
+ with pybigtools.open(str(source)) as reader:
236
+ return dict(reader.chroms())
237
+
238
+
239
+ def _convert(tasks: Sequence[tuple[Path, Path, str, Mapping[str, str]]], count: int) -> list[ContigSummary | None]:
240
+ if count == 1:
241
+ return [_convert_contig(*task) for task in tasks]
242
+ # `spawn` starts each worker from a fresh interpreter: forking a process with threads running can deadlock it.
243
+ context = multiprocessing.get_context("spawn")
244
+ with ProcessPoolExecutor(max_workers=count, mp_context=context) as pool:
245
+ futures = [pool.submit(_convert_contig, *task) for task in tasks]
246
+ try:
247
+ return [future.result() for future in futures]
248
+ except BaseException:
249
+ for future in futures:
250
+ future.cancel()
251
+ raise
252
+
253
+
254
+ def _convert_contig(source: Path, output: Path, contig: str, metadata: Mapping[str, str]) -> ContigSummary | None:
255
+ import pybigtools # noqa: PLC0415 - the `build` extra
256
+
257
+ with pybigtools.open(str(source)) as reader:
258
+ return write_contig(output, contig, reader.records(contig), metadata)
259
+
260
+
261
+ def write_contig(
262
+ output: Path, contig: str, records: Iterable[tuple[int, int, float]], metadata: Mapping[str, str]
263
+ ) -> ContigSummary | None:
264
+ """Write `contig`'s rows from its bigWig records to `output/chr=<contig>/`; return `None` if it has none.
265
+
266
+ `records` are `(start, end, value)`, 0-based and half-open, in order, as pybigtools reads them; `metadata` is
267
+ `track_metadata`'s. The file is written under a ``.partial`` name and renamed when it is complete.
268
+ """
269
+ writer = PartitionedWriter(
270
+ output,
271
+ _STEM,
272
+ schema=SCHEMA,
273
+ partition_column="chr",
274
+ row_group_rows=ROW_GROUP_ROWS,
275
+ metadata=metadata,
276
+ parquet_options=PARQUET_OPTIONS,
277
+ )
278
+ rows = 0
279
+ try:
280
+ for batch in contig_batches(contig, records):
281
+ writer.append_batch(contig, batch)
282
+ rows += batch.num_rows
283
+ files = writer.commit()
284
+ except BaseException:
285
+ writer.abort()
286
+ raise
287
+ if not files:
288
+ return None
289
+ [file] = files
290
+ return ContigSummary(
291
+ contig=contig,
292
+ rows=rows,
293
+ file=file,
294
+ bytes=file.stat().st_size,
295
+ row_groups=pq.ParquetFile(file).metadata.num_row_groups,
296
+ )
297
+
298
+
299
+ def main() -> None:
300
+ """Run the conservation track builder command."""
301
+ parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0])
302
+ parser.add_argument("--track", required=True, choices=sorted(TRACKS), help="The track to build")
303
+ parser.add_argument("--input", required=True, help="The track's upstream bigWig file, downloaded locally")
304
+ parser.add_argument("--output", required=True, help="The local directory to write the dataset to")
305
+ parser.add_argument("--overwrite", action="store_true", help="Replace a previous build in --output")
306
+ parser.add_argument("--workers", type=int, help="Contigs to convert at once, in processes (default: CPU count)")
307
+ args = parser.parse_args()
308
+ summary = build_dataset(TRACKS[args.track], args.input, args.output, overwrite=args.overwrite, workers=args.workers)
309
+ for contig in summary.contigs:
310
+ print(f"{contig.contig}\t{contig.rows}") # noqa: T201 - command-line report
311
+ print( # noqa: T201
312
+ f"wrote {summary.rows} rows of {summary.track} in {len(summary.files)} files and {summary.row_groups} row "
313
+ f"groups ({summary.bytes / 1e9:.2f} GB) to {args.output}; skipped {summary.skipped_contigs} non-primary "
314
+ f"contigs; {summary.seconds:.0f} s"
315
+ )
316
+
317
+
318
+ if __name__ == "__main__":
319
+ main()
File without changes