twobitreader-rs 0.2.2__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/CHANGELOG.md +43 -6
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/Cargo.lock +23 -23
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/Cargo.toml +12 -7
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/PKG-INFO +3 -3
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/README.md +34 -16
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/pyproject.toml +3 -2
- twobitreader_rs-0.3.0/src/bin/twobitreader.rs +441 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/src/lib.rs +67 -26
- twobitreader_rs-0.3.0/src/seq_index_map.rs +147 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/LICENSE-APACHE +0 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/LICENSE-MIT +0 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/README-PY.md +0 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/custom.css +0 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/py.typed +0 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/src/decode.rs +0 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/src/prefetch.rs +0 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/src/python.rs +0 -0
- {twobitreader_rs-0.2.2 → twobitreader_rs-0.3.0}/twobitreader_rs.pyi +0 -0
|
@@ -4,10 +4,45 @@ All notable changes to this project are documented here.
|
|
|
4
4
|
|
|
5
5
|
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this
|
|
6
6
|
project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). The Rust
|
|
7
|
-
crate `twobitreader` and the Python package `
|
|
7
|
+
crate `twobitreader` and the Python package `twobitreader-rs` share a version number.
|
|
8
8
|
|
|
9
9
|
## [Unreleased]
|
|
10
10
|
|
|
11
|
+
## [0.3.0] - 2026-11-06
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- `twobitreader` command-line tool, behind the `cli` feature, a drop-in replacement for the core
|
|
16
|
+
features of UCSC `twoBitToFa` that is several times faster on BED extraction (see `BENCH.md`). Its
|
|
17
|
+
`-threads=N` option extracts a `-bed` or `-seqList` file in parallel, with output in input order,
|
|
18
|
+
and its `-prefetch` option asks the operating system to page in the needed data first, which
|
|
19
|
+
greatly speeds up a 2bit file that is not already in the page cache.
|
|
20
|
+
- `TwobitReader::try_get_into`, a version of `get_into` that returns an error instead of panicking.
|
|
21
|
+
- Python 3.15 wheels, including free-threaded 3.15t.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
|
|
25
|
+
- Dropped support for Python 3.9, which is end-of-life.
|
|
26
|
+
- Sequence names chr1-22/X/Y/M are now looked up without hashing, which is ~2.5x faster per lookup.
|
|
27
|
+
Other names fall back to a hash map as before.
|
|
28
|
+
- Narrowed the version requirements of dependencies from `"0"` to the ranges that were tested, and
|
|
29
|
+
committed `Cargo.lock`.
|
|
30
|
+
|
|
31
|
+
## [0.2.3] - 2026-11-03
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
|
|
35
|
+
- `TwobitReader::open` no longer rejects a valid 2bit file that contains exactly one sequence
|
|
36
|
+
record. The record's data starts immediately after the index, so its offset equals the end
|
|
37
|
+
of the index, which the offset check wrongly treated as invalid.
|
|
38
|
+
- Missing word and a misplaced sentence in the Benchmarks section of the documentation.
|
|
39
|
+
|
|
40
|
+
### Changed
|
|
41
|
+
|
|
42
|
+
- README.md is now generated from the crate documentation with `cargo rdme`, and CI checks
|
|
43
|
+
that it is up to date. The badges moved from the crate documentation into README.md, so
|
|
44
|
+
they no longer appear on docs.rs.
|
|
45
|
+
|
|
11
46
|
## [0.2.2] - 2026-10-01
|
|
12
47
|
|
|
13
48
|
### Added
|
|
@@ -16,7 +51,7 @@ crate `twobitreader` and the Python package `twobitreader_rs` share a version nu
|
|
|
16
51
|
|
|
17
52
|
## [0.2.1] - 2026-10-01
|
|
18
53
|
|
|
19
|
-
First release of `
|
|
54
|
+
First release of `twobitreader-rs` on PyPI. There are no library changes; 0.2.0 reached
|
|
20
55
|
crates.io before the wheel-building workflow was in place, so it was never published for
|
|
21
56
|
Python.
|
|
22
57
|
|
|
@@ -31,13 +66,13 @@ Python.
|
|
|
31
66
|
|
|
32
67
|
## [0.2.0] - 2026-10-01
|
|
33
68
|
|
|
34
|
-
`twobitreader` is now available from Python as **`
|
|
69
|
+
`twobitreader` is now available from Python as **`twobitreader-rs`**, built with PyO3 and
|
|
35
70
|
published as a wheel. The Python API mirrors the Rust one, with speed close to pure Rust.
|
|
36
71
|
|
|
37
72
|
### Added
|
|
38
73
|
|
|
39
|
-
- **Python bindings** (`twobitreader_rs`), exposing `TwobitReader` and
|
|
40
|
-
Available for CPython 3.9+ on Linux, macOS and Windows.
|
|
74
|
+
- **Python bindings** (`import twobitreader_rs`), exposing `TwobitReader` and
|
|
75
|
+
`reverse_complement`. Available for CPython 3.9+ on Linux, macOS and Windows.
|
|
41
76
|
- **Free-threaded CPython support.** The extension is marked `gil_used = false` and the
|
|
42
77
|
reader is immutable and shareable. A thread pool over batches scales across cores on a
|
|
43
78
|
free-threaded build.
|
|
@@ -71,7 +106,9 @@ extraction with `get_batch`, exon concatenation with `concat`, reverse complemen
|
|
|
71
106
|
`prefetch` for cold files. Sequence records are parsed lazily so that opening a large file
|
|
72
107
|
stays cheap. No git tag was made for this version.
|
|
73
108
|
|
|
74
|
-
[Unreleased]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.
|
|
109
|
+
[Unreleased]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.3.0...HEAD
|
|
110
|
+
[0.3.0]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.2.3...v0.3.0
|
|
111
|
+
[0.2.3]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.2.2...v0.2.3
|
|
75
112
|
[0.2.2]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.2.1...v0.2.2
|
|
76
113
|
[0.2.1]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.2.0...v0.2.1
|
|
77
114
|
[0.2.0]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.1.1...v0.2.0
|
|
@@ -22,9 +22,9 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
|
|
22
22
|
|
|
23
23
|
[[package]]
|
|
24
24
|
name = "crc32fast"
|
|
25
|
-
version = "1.5.
|
|
25
|
+
version = "1.5.4"
|
|
26
26
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
27
|
-
checksum = "
|
|
27
|
+
checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04"
|
|
28
28
|
dependencies = [
|
|
29
29
|
"cfg-if",
|
|
30
30
|
]
|
|
@@ -56,9 +56,9 @@ checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
|
|
56
56
|
|
|
57
57
|
[[package]]
|
|
58
58
|
name = "either"
|
|
59
|
-
version = "1.
|
|
59
|
+
version = "1.19.0"
|
|
60
60
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
61
|
-
checksum = "
|
|
61
|
+
checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
|
|
62
62
|
|
|
63
63
|
[[package]]
|
|
64
64
|
name = "flate2"
|
|
@@ -88,15 +88,15 @@ dependencies = [
|
|
|
88
88
|
|
|
89
89
|
[[package]]
|
|
90
90
|
name = "libc"
|
|
91
|
-
version = "0.2.
|
|
91
|
+
version = "0.2.191"
|
|
92
92
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
93
|
-
checksum = "
|
|
93
|
+
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
|
94
94
|
|
|
95
95
|
[[package]]
|
|
96
96
|
name = "memmap2"
|
|
97
|
-
version = "0.9.
|
|
97
|
+
version = "0.9.12"
|
|
98
98
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
99
|
-
checksum = "
|
|
99
|
+
checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157"
|
|
100
100
|
dependencies = [
|
|
101
101
|
"libc",
|
|
102
102
|
]
|
|
@@ -134,9 +134,9 @@ dependencies = [
|
|
|
134
134
|
|
|
135
135
|
[[package]]
|
|
136
136
|
name = "pyo3"
|
|
137
|
-
version = "0.
|
|
137
|
+
version = "0.30.1"
|
|
138
138
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
139
|
-
checksum = "
|
|
139
|
+
checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
|
|
140
140
|
dependencies = [
|
|
141
141
|
"libc",
|
|
142
142
|
"once_cell",
|
|
@@ -148,18 +148,18 @@ dependencies = [
|
|
|
148
148
|
|
|
149
149
|
[[package]]
|
|
150
150
|
name = "pyo3-build-config"
|
|
151
|
-
version = "0.
|
|
151
|
+
version = "0.30.1"
|
|
152
152
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
153
|
-
checksum = "
|
|
153
|
+
checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
|
|
154
154
|
dependencies = [
|
|
155
155
|
"target-lexicon",
|
|
156
156
|
]
|
|
157
157
|
|
|
158
158
|
[[package]]
|
|
159
159
|
name = "pyo3-ffi"
|
|
160
|
-
version = "0.
|
|
160
|
+
version = "0.30.1"
|
|
161
161
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
162
|
-
checksum = "
|
|
162
|
+
checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
|
|
163
163
|
dependencies = [
|
|
164
164
|
"libc",
|
|
165
165
|
"pyo3-build-config",
|
|
@@ -167,9 +167,9 @@ dependencies = [
|
|
|
167
167
|
|
|
168
168
|
[[package]]
|
|
169
169
|
name = "pyo3-macros"
|
|
170
|
-
version = "0.
|
|
170
|
+
version = "0.30.1"
|
|
171
171
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
172
|
-
checksum = "
|
|
172
|
+
checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
|
|
173
173
|
dependencies = [
|
|
174
174
|
"proc-macro2",
|
|
175
175
|
"pyo3-macros-backend",
|
|
@@ -179,9 +179,9 @@ dependencies = [
|
|
|
179
179
|
|
|
180
180
|
[[package]]
|
|
181
181
|
name = "pyo3-macros-backend"
|
|
182
|
-
version = "0.
|
|
182
|
+
version = "0.30.1"
|
|
183
183
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
184
|
-
checksum = "
|
|
184
|
+
checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
|
|
185
185
|
dependencies = [
|
|
186
186
|
"heck",
|
|
187
187
|
"proc-macro2",
|
|
@@ -238,9 +238,9 @@ checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
|
|
238
238
|
|
|
239
239
|
[[package]]
|
|
240
240
|
name = "syn"
|
|
241
|
-
version = "
|
|
241
|
+
version = "3.0.9"
|
|
242
242
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
243
|
-
checksum = "
|
|
243
|
+
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
244
244
|
dependencies = [
|
|
245
245
|
"proc-macro2",
|
|
246
246
|
"quote",
|
|
@@ -255,7 +255,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
255
255
|
|
|
256
256
|
[[package]]
|
|
257
257
|
name = "twobitreader"
|
|
258
|
-
version = "0.
|
|
258
|
+
version = "0.3.0"
|
|
259
259
|
dependencies = [
|
|
260
260
|
"byteorder",
|
|
261
261
|
"flate2",
|
|
@@ -271,9 +271,9 @@ dependencies = [
|
|
|
271
271
|
|
|
272
272
|
[[package]]
|
|
273
273
|
name = "unicode-ident"
|
|
274
|
-
version = "1.0.
|
|
274
|
+
version = "1.0.27"
|
|
275
275
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
276
|
-
checksum = "
|
|
276
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
277
277
|
|
|
278
278
|
[[package]]
|
|
279
279
|
name = "windows-link"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "twobitreader"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
authors = ["Andrew Delong <andrew.delong@gmail.com>"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -13,21 +13,22 @@ exclude = ["tests/", "benches/", "doc/", "*.code-workspace", ".*", "BENCH.md"]
|
|
|
13
13
|
|
|
14
14
|
[features]
|
|
15
15
|
python = ["dep:pyo3", "dep:scopeguard"]
|
|
16
|
+
cli = []
|
|
16
17
|
|
|
17
18
|
[dependencies]
|
|
18
19
|
byteorder = "1"
|
|
19
|
-
memmap2 = "0"
|
|
20
|
-
seq-macro = "0"
|
|
21
|
-
pyo3 = { version = "0", optional = true }
|
|
20
|
+
memmap2 = ">=0.1, <0.10"
|
|
21
|
+
seq-macro = ">=0.1.3, <0.4"
|
|
22
|
+
pyo3 = { version = ">=0.28, <0.31", optional = true }
|
|
22
23
|
scopeguard = { version = "1", optional = true}
|
|
23
24
|
|
|
24
25
|
# Apple targets need fcntl(F_RDADVISE) to prefetch file ranges; madvise does not work there.
|
|
25
26
|
[target.'cfg(target_vendor = "apple")'.dependencies]
|
|
26
|
-
libc = "0"
|
|
27
|
+
libc = "0.2.2"
|
|
27
28
|
|
|
28
29
|
# Windows targets need PrefetchVirtualMemory to prefetch file ranges.
|
|
29
30
|
[target.'cfg(windows)'.dependencies]
|
|
30
|
-
windows-sys = { version = "0", features = ["Win32_System_Memory", "Win32_System_Threading"] }
|
|
31
|
+
windows-sys = { version = "0.61", features = ["Win32_System_Memory", "Win32_System_Threading"] }
|
|
31
32
|
|
|
32
33
|
[dev-dependencies]
|
|
33
34
|
flate2 = "1"
|
|
@@ -36,7 +37,11 @@ rayon = "1"
|
|
|
36
37
|
|
|
37
38
|
# Cold benchmarks use fcntl(F_NOCACHE) to write an uncached copy of the 2bit file.
|
|
38
39
|
[target.'cfg(target_vendor = "apple")'.dev-dependencies]
|
|
39
|
-
libc = "0"
|
|
40
|
+
libc = "0.2.2"
|
|
41
|
+
|
|
42
|
+
[[bin]]
|
|
43
|
+
name = "twobitreader"
|
|
44
|
+
required-features = ["cli"]
|
|
40
45
|
|
|
41
46
|
[package.metadata.docs.rs]
|
|
42
47
|
rustdoc-args = ["--extend-css", "custom.css"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: twobitreader-rs
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
5
5
|
Classifier: Intended Audience :: Science/Research
|
|
6
6
|
Classifier: Operating System :: POSIX :: Linux
|
|
@@ -10,12 +10,12 @@ Classifier: Development Status :: 4 - Beta
|
|
|
10
10
|
Classifier: Programming Language :: Rust
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
12
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.10
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
19
19
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
20
|
Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
|
|
21
21
|
Classifier: Typing :: Typed
|
|
@@ -25,7 +25,7 @@ License-File: LICENSE-MIT
|
|
|
25
25
|
License-File: LICENSE-APACHE
|
|
26
26
|
Summary: Python bindings for the twobitreader Rust crate
|
|
27
27
|
License-Expression: MIT OR Apache-2.0
|
|
28
|
-
Requires-Python: >=3.
|
|
28
|
+
Requires-Python: >=3.10
|
|
29
29
|
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
30
30
|
Project-URL: Repository, https://github.com/andrewdelong/twobitreader-rust
|
|
31
31
|
|
|
@@ -1,21 +1,22 @@
|
|
|
1
1
|
# twobitreader
|
|
2
2
|
|
|
3
|
+
[](https://github.com/andrewdelong/twobitreader-rust/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
[](https://anaconda.org/conda-forge/twobitreader-rs)
|
|
8
|
+
[](https://pypi.org/project/twobitreader-rs/)
|
|
9
|
+
|
|
10
|
+
<!-- cargo-rdme start -->
|
|
3
11
|
|
|
4
12
|
This crate provides fast DNA sequence extraction from 2bit files, a
|
|
5
13
|
[standard format](http://genome.ucsc.edu/FAQ/FAQformat.html#format7) in bioinformatics.
|
|
6
14
|
|
|
7
|
-
The motivation for `twobitreader` is speed; see benchmarks below.
|
|
8
|
-
It is also available as a Python package named `twobitreader-rs`.
|
|
9
|
-
|
|
10
15
|
The focus is raw reading from 2bit, but fast concatenation and reverse-complement methods
|
|
11
16
|
are also provided to make higher-level use cases easier.
|
|
12
17
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
[](https://anaconda.org/conda-forge/twobitreader-rs)
|
|
18
|
-
[](https://pypi.org/project/twobitreader-rs/)
|
|
18
|
+
The crate is also available as a Python package (`twobitreader-rs`) or as a
|
|
19
|
+
terminal command replacing `twoBitToFa`.
|
|
19
20
|
|
|
20
21
|
## Examples
|
|
21
22
|
|
|
@@ -78,6 +79,16 @@ tbr.prefetch(&exons); // Ask the operating system to start paging th
|
|
|
78
79
|
let seqs = tbr.get_batch(&exons); // Access the memory as it arrives.
|
|
79
80
|
```
|
|
80
81
|
|
|
82
|
+
## Command line
|
|
83
|
+
|
|
84
|
+
The `cli` feature builds a `twobitreader` command that is a drop-in replacement for the core features
|
|
85
|
+
of UCSC `twoBitToFa`:
|
|
86
|
+
```console
|
|
87
|
+
cargo install twobitreader --features cli
|
|
88
|
+
twobitreader -seq=chr1 -start=10000 -end=10005 hg38.2bit stdout
|
|
89
|
+
twobitreader -bed=exons.bed -threads=16 hg38.2bit out.fa
|
|
90
|
+
```
|
|
91
|
+
|
|
81
92
|
## Benchmarks
|
|
82
93
|
|
|
83
94
|
Two tasks were benchmarked:
|
|
@@ -89,13 +100,14 @@ Speed depends on parallelism and page cache (hot vs cold):
|
|
|
89
100
|
- **cold** runs represent a first run of a genomics pipeline, bound by disk speed;
|
|
90
101
|
- **prefetch** runs are cold but with a `prefetch` call preceding extraction.
|
|
91
102
|
|
|
92
|
-
The table below shows running times in milliseconds. Experimental details are `BENCH.md`.
|
|
93
|
-
This crate provides
|
|
103
|
+
The table below shows running times in milliseconds. Experimental details are in `BENCH.md`.
|
|
104
|
+
This crate provides the top three rows: *twobitreader*, *twobitreader-rs*, *twobitreader-cli*.
|
|
94
105
|
|
|
95
106
|
| EXONS | 1-thread / hot | 1-thread / cold | 1-thread / prefetch | 16-thread / hot | 16-thread / cold |
|
|
96
107
|
|---|---:|---:|---:|---:|---:|
|
|
97
108
|
| **twobitreader** (rs) | 90 | 1,700 | 180 | 10 | 190 |
|
|
98
|
-
| **
|
|
109
|
+
| **twobitreader-rs** (rs, py) | 100 | 1,900 | 190 | *27 | *210 |
|
|
110
|
+
| **twobitreader-cli** (rs) | 150 | 2,200 | 280 | 55 | 260 |
|
|
99
111
|
| **py2bit** (c, py) | 220 | 2,800 | n/a | n/a | n/a |
|
|
100
112
|
| **GenomeKit** (cpp, py) | 340 | 2,400 | n/a | n/a | n/a |
|
|
101
113
|
| **twobit** (rs) | 390 | 3,500 | n/a | n/a | n/a |
|
|
@@ -106,7 +118,8 @@ This crate provides *twobitreader* (pure rust) and *twobitreader_rs* (python wra
|
|
|
106
118
|
| TRANSCRIPTS | 1-thread / hot | 1-thread / cold | 1-thread / prefetch | 16-thread / hot | 16-thread / cold |
|
|
107
119
|
|---|---:|---:|---:|---:|---:|
|
|
108
120
|
| **twobitreader** (rs) | 140 | 1,700 | 230 | 13 | 190 |
|
|
109
|
-
| **
|
|
121
|
+
| **twobitreader-rs** (rs, py) | 180 | 2,100 | 310 | *27 | *220 |
|
|
122
|
+
| **twobitreader-cli** (rs) | 250 | 2,300 | 340 | 75 | 300 |
|
|
110
123
|
| **py2bit** (c, py) | 490 | 3,600 | n/a | n/a | n/a |
|
|
111
124
|
| **GenomeKit** (cpp, py) | 720 | 2,900 | n/a | n/a | n/a |
|
|
112
125
|
| **twobit** (rs) | 930 | 1,900 | n/a | n/a | n/a |
|
|
@@ -126,8 +139,13 @@ Entries marked * were run in free-threaded Python.
|
|
|
126
139
|
|
|
127
140
|
## Use of AI
|
|
128
141
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
142
|
+
The Rust library was written primarily by hand.
|
|
143
|
+
|
|
144
|
+
Claude was used to write the CLI and its tests+benchmark, to generated the OS-specific
|
|
145
|
+
prefetch loops, to improve handling of corrupt or malicious files, to document the
|
|
146
|
+
Python bindings and to write their tests+benchmark, to improved error checking
|
|
147
|
+
and propagation to Python more broadly, and to generated the CI configurations.
|
|
148
|
+
|
|
149
|
+
<!-- cargo-rdme end -->
|
|
132
150
|
|
|
133
151
|
License: MIT OR Apache-2.0
|
|
@@ -6,7 +6,7 @@ build-backend = "maturin"
|
|
|
6
6
|
name = "twobitreader-rs"
|
|
7
7
|
description = "Python bindings for the twobitreader Rust crate"
|
|
8
8
|
readme = "README-PY.md"
|
|
9
|
-
requires-python = ">=3.
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
10
|
classifiers = [
|
|
11
11
|
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
|
12
12
|
"Intended Audience :: Science/Research",
|
|
@@ -17,12 +17,12 @@ classifiers = [
|
|
|
17
17
|
"Programming Language :: Rust",
|
|
18
18
|
"Programming Language :: Python :: 3",
|
|
19
19
|
"Programming Language :: Python :: 3 :: Only",
|
|
20
|
-
"Programming Language :: Python :: 3.9",
|
|
21
20
|
"Programming Language :: Python :: 3.10",
|
|
22
21
|
"Programming Language :: Python :: 3.11",
|
|
23
22
|
"Programming Language :: Python :: 3.12",
|
|
24
23
|
"Programming Language :: Python :: 3.13",
|
|
25
24
|
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Programming Language :: Python :: 3.15",
|
|
26
26
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
27
27
|
"Programming Language :: Python :: Free Threading :: 2 - Beta",
|
|
28
28
|
"Typing :: Typed",
|
|
@@ -41,4 +41,5 @@ tests = [
|
|
|
41
41
|
|
|
42
42
|
[tool.maturin]
|
|
43
43
|
features = ["python"]
|
|
44
|
+
bindings = "pyo3"
|
|
44
45
|
module-name = "twobitreader_rs"
|
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
//! `twobitreader`: a command-line replacement for UCSC `twoBitToFa`.
|
|
2
|
+
//!
|
|
3
|
+
//! Build with `cargo build --release --features cli`.
|
|
4
|
+
|
|
5
|
+
use std::fmt::Arguments;
|
|
6
|
+
use std::fs::File;
|
|
7
|
+
use std::io::{self, BufRead, BufReader, BufWriter, Write};
|
|
8
|
+
use std::ops::Range;
|
|
9
|
+
use std::process::ExitCode;
|
|
10
|
+
use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
|
|
11
|
+
use std::sync::{Condvar, Mutex};
|
|
12
|
+
|
|
13
|
+
use twobitreader::{TwobitReader, reverse_complement};
|
|
14
|
+
|
|
15
|
+
const LINE_WIDTH: usize = 50;
|
|
16
|
+
|
|
17
|
+
const USAGE: &str = "\
|
|
18
|
+
twobitreader - convert all or part of a .2bit file to FASTA (compatible with twoBitToFa)
|
|
19
|
+
usage:
|
|
20
|
+
twobitreader [options] input.2bit output.fa
|
|
21
|
+
Use 'stdout' as output.fa to write to standard output.
|
|
22
|
+
options:
|
|
23
|
+
-seq=name Restrict output to this sequence. May be name:start-end.
|
|
24
|
+
-start=N Start at this position in the sequence (zero-based). Requires -seq.
|
|
25
|
+
-end=N End at this position in the sequence (non-inclusive). Requires -seq.
|
|
26
|
+
Default is the end of the sequence.
|
|
27
|
+
-seqList=file File of sequence specs, one per line: name or name:start-end,
|
|
28
|
+
with half-open zero-based coordinates.
|
|
29
|
+
-noMask Convert sequence to all upper case. Default keeps lowercase mask.
|
|
30
|
+
-bed=file Extract the regions in a BED file (BED3 to BED12). BED12 blocks are
|
|
31
|
+
concatenated, excluding introns, and minus-strand
|
|
32
|
+
records are reverse-complemented.
|
|
33
|
+
-bedPos With -bed, name each record chrom:start-end instead of the BED name.
|
|
34
|
+
-threads=N With -bed or -seqList, extract with N threads (0 = one per core). The
|
|
35
|
+
whole list is read into memory first. Output order is unchanged.
|
|
36
|
+
-prefetch With -bed or -seqList, first ask the operating system to page in all the
|
|
37
|
+
sequence data the list needs. Speeds up a 2bit file that is not in the
|
|
38
|
+
page cache; slightly slows one that is. The whole list is read first.
|
|
39
|
+
-h, -help Show this message.
|
|
40
|
+
Sequences are written 50 bases per line. A record extracted with a range
|
|
41
|
+
is named name:start-end.
|
|
42
|
+
";
|
|
43
|
+
|
|
44
|
+
#[derive(Default)]
|
|
45
|
+
struct Options {
|
|
46
|
+
seq: Option<String>,
|
|
47
|
+
start: Option<usize>,
|
|
48
|
+
end: Option<usize>,
|
|
49
|
+
seq_list: Option<String>,
|
|
50
|
+
bed: Option<String>,
|
|
51
|
+
bed_pos: bool,
|
|
52
|
+
no_mask: bool,
|
|
53
|
+
threads: Option<usize>, // None = sequential and streaming; Some(0) = one per core.
|
|
54
|
+
prefetch: bool,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
fn main() -> ExitCode {
|
|
58
|
+
// The library panics on corrupt files that pass validation at open time; report cleanly.
|
|
59
|
+
std::panic::set_hook(Box::new(|info| {
|
|
60
|
+
let msg = info
|
|
61
|
+
.payload()
|
|
62
|
+
.downcast_ref::<String>()
|
|
63
|
+
.map(String::as_str)
|
|
64
|
+
.or_else(|| info.payload().downcast_ref::<&str>().copied())
|
|
65
|
+
.unwrap_or("unexpected failure");
|
|
66
|
+
eprintln!("twobitreader: {msg}");
|
|
67
|
+
std::process::exit(1);
|
|
68
|
+
}));
|
|
69
|
+
match run() {
|
|
70
|
+
Ok(()) => ExitCode::SUCCESS,
|
|
71
|
+
Err(e) if e.kind() == io::ErrorKind::BrokenPipe => ExitCode::SUCCESS,
|
|
72
|
+
Err(e) => {
|
|
73
|
+
eprintln!("twobitreader: {e}");
|
|
74
|
+
ExitCode::FAILURE
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
fn err<T>(msg: impl Into<String>) -> io::Result<T> {
|
|
80
|
+
Err(io::Error::other(msg.into()))
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
fn usage_error<T>(msg: &str) -> io::Result<T> {
|
|
84
|
+
err(format!("{msg}\n\n{USAGE}"))
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
fn parse_args() -> io::Result<Option<(Options, String, String)>> {
|
|
88
|
+
let mut opts = Options::default();
|
|
89
|
+
let mut positional = Vec::new();
|
|
90
|
+
for arg in std::env::args().skip(1) {
|
|
91
|
+
let Some(body) = arg.strip_prefix('-').filter(|b| !b.is_empty()) else {
|
|
92
|
+
positional.push(arg);
|
|
93
|
+
continue;
|
|
94
|
+
};
|
|
95
|
+
let body = body.strip_prefix('-').unwrap_or(body);
|
|
96
|
+
let (key, value) = match body.split_once('=') {
|
|
97
|
+
Some((k, v)) => (k, Some(v.to_string())),
|
|
98
|
+
None => (body, None),
|
|
99
|
+
};
|
|
100
|
+
let need = |v: Option<String>| v.map_or_else(|| usage_error(&format!("option -{key} requires =value")), Ok);
|
|
101
|
+
let num = |v: Option<String>| -> io::Result<usize> {
|
|
102
|
+
let v = need(v)?;
|
|
103
|
+
v.parse().or_else(|_| usage_error(&format!("bad number for -{key}: {v}")))
|
|
104
|
+
};
|
|
105
|
+
match key {
|
|
106
|
+
"h" | "help" => return Ok(None),
|
|
107
|
+
"seq" => opts.seq = Some(need(value)?),
|
|
108
|
+
"start" => opts.start = Some(num(value)?),
|
|
109
|
+
"end" => opts.end = Some(num(value)?),
|
|
110
|
+
"seqList" => opts.seq_list = Some(need(value)?),
|
|
111
|
+
"bed" => opts.bed = Some(need(value)?),
|
|
112
|
+
"bedPos" => opts.bed_pos = true,
|
|
113
|
+
"noMask" => opts.no_mask = true,
|
|
114
|
+
"threads" => opts.threads = Some(num(value)?),
|
|
115
|
+
"prefetch" => opts.prefetch = true,
|
|
116
|
+
"bpt" | "udcDir" => return usage_error(&format!("option -{key} is not supported")),
|
|
117
|
+
_ => return usage_error(&format!("unrecognized option -{key}")),
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
let [input, output] =
|
|
121
|
+
<[String; 2]>::try_from(positional).or_else(|_| usage_error("expected two arguments: input.2bit output.fa"))?;
|
|
122
|
+
let modes = [opts.seq.is_some(), opts.seq_list.is_some(), opts.bed.is_some()];
|
|
123
|
+
if modes.iter().filter(|&&m| m).count() > 1 {
|
|
124
|
+
return usage_error("-seq, -seqList and -bed are mutually exclusive");
|
|
125
|
+
}
|
|
126
|
+
if (opts.start.is_some() || opts.end.is_some()) && opts.seq.is_none() {
|
|
127
|
+
eprintln!("twobitreader: warning: -start and -end are ignored without -seq");
|
|
128
|
+
opts.start = None;
|
|
129
|
+
opts.end = None;
|
|
130
|
+
}
|
|
131
|
+
if opts.seq.as_deref().is_some_and(|s| parse_spec(s).1.is_some()) && (opts.start.is_some() || opts.end.is_some()) {
|
|
132
|
+
return usage_error("-seq=name:start-end cannot be combined with -start or -end");
|
|
133
|
+
}
|
|
134
|
+
if opts.bed_pos && opts.bed.is_none() {
|
|
135
|
+
return usage_error("-bedPos requires -bed");
|
|
136
|
+
}
|
|
137
|
+
if opts.threads.is_some() && opts.bed.is_none() && opts.seq_list.is_none() {
|
|
138
|
+
return usage_error("-threads requires -bed or -seqList");
|
|
139
|
+
}
|
|
140
|
+
if opts.prefetch && opts.bed.is_none() && opts.seq_list.is_none() {
|
|
141
|
+
return usage_error("-prefetch requires -bed or -seqList");
|
|
142
|
+
}
|
|
143
|
+
Ok(Some((opts, input, output)))
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
fn run() -> io::Result<()> {
|
|
147
|
+
let Some((opts, input, output)) = parse_args()? else {
|
|
148
|
+
print!("{USAGE}");
|
|
149
|
+
return Ok(());
|
|
150
|
+
};
|
|
151
|
+
let (input, inline_spec) = match input.find(".2bit:") {
|
|
152
|
+
Some(i) if !std::path::Path::new(&input).exists() => {
|
|
153
|
+
(input[..i + 5].to_string(), Some(input[i + 6..].to_string()))
|
|
154
|
+
}
|
|
155
|
+
_ => (input, None),
|
|
156
|
+
};
|
|
157
|
+
if inline_spec.is_some() && (opts.seq.is_some() || opts.seq_list.is_some() || opts.bed.is_some()) {
|
|
158
|
+
return usage_error("input.2bit:name cannot be combined with -seq, -seqList or -bed");
|
|
159
|
+
}
|
|
160
|
+
let tbr = if opts.no_mask { TwobitReader::open(&input) } else { TwobitReader::open_masked(&input) }
|
|
161
|
+
.map_err(|e| io::Error::new(e.kind(), format!("{input}: {e}")))?;
|
|
162
|
+
|
|
163
|
+
let dst: Box<dyn Write> = if output == "stdout" {
|
|
164
|
+
Box::new(io::stdout().lock())
|
|
165
|
+
} else {
|
|
166
|
+
Box::new(File::create(&output).map_err(|e| io::Error::new(e.kind(), format!("{output}: {e}")))?)
|
|
167
|
+
};
|
|
168
|
+
let mut out = BufWriter::with_capacity(1 << 20, dst);
|
|
169
|
+
|
|
170
|
+
let mut seq = String::new(); // Reused for every record, to avoid reallocating.
|
|
171
|
+
if let Some(path) = &opts.bed {
|
|
172
|
+
if opts.prefetch {
|
|
173
|
+
prefetch_list(&tbr, path, true)?;
|
|
174
|
+
}
|
|
175
|
+
let process = |line: &str, seq: &mut String, out: &mut Vec<u8>| {
|
|
176
|
+
if is_skippable(line, true) {
|
|
177
|
+
return Ok(());
|
|
178
|
+
}
|
|
179
|
+
write_bed_record(&tbr, &input, line, opts.bed_pos, seq, out)
|
|
180
|
+
};
|
|
181
|
+
run_lines(path, opts.threads, &process, &mut out)?;
|
|
182
|
+
} else if let Some(path) = &opts.seq_list {
|
|
183
|
+
if opts.prefetch {
|
|
184
|
+
prefetch_list(&tbr, path, false)?;
|
|
185
|
+
}
|
|
186
|
+
let process = |line: &str, seq: &mut String, out: &mut Vec<u8>| {
|
|
187
|
+
if is_skippable(line, false) {
|
|
188
|
+
return Ok(());
|
|
189
|
+
}
|
|
190
|
+
write_spec(&tbr, &input, line.trim(), None, None, seq, out)
|
|
191
|
+
};
|
|
192
|
+
run_lines(path, opts.threads, &process, &mut out)?;
|
|
193
|
+
} else if let Some(spec) = inline_spec.as_ref().or(opts.seq.as_ref()) {
|
|
194
|
+
write_spec(&tbr, &input, spec, opts.start, opts.end, &mut seq, &mut out)?;
|
|
195
|
+
} else {
|
|
196
|
+
for name in tbr.iter_names() {
|
|
197
|
+
tbr.get_into(name, 0, tbr.seq_len(name), &mut seq);
|
|
198
|
+
write_record(&mut out, format_args!("{name}"), &seq)?;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
out.flush()
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Asks the operating system to page in the sequence data that every line of the BED or seqList file will read.
|
|
205
|
+
// Lines that are malformed or out of range are left for the extraction to report.
|
|
206
|
+
fn prefetch_list(tbr: &TwobitReader, path: &str, bed: bool) -> io::Result<()> {
|
|
207
|
+
let text = std::fs::read_to_string(path).map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?;
|
|
208
|
+
let mut ranges = Vec::new();
|
|
209
|
+
for line in text.lines().filter(|line| !is_skippable(line, bed)) {
|
|
210
|
+
let (name, range) = if bed {
|
|
211
|
+
let mut cols = line.split_ascii_whitespace();
|
|
212
|
+
let (Some(chrom), Some(start), Some(end)) = (cols.next(), cols.next(), cols.next()) else { continue };
|
|
213
|
+
let (Ok(start), Ok(end)) = (start.parse(), end.parse()) else { continue };
|
|
214
|
+
(chrom, Some((start, end)))
|
|
215
|
+
} else {
|
|
216
|
+
parse_spec(line.trim())
|
|
217
|
+
};
|
|
218
|
+
if tbr.contains_name(name) {
|
|
219
|
+
let len = tbr.seq_len(name);
|
|
220
|
+
let (start, end) = range.unwrap_or((0, len));
|
|
221
|
+
if start <= end && end <= len {
|
|
222
|
+
ranges.push((name, start, end));
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
tbr.prefetch(&ranges);
|
|
227
|
+
Ok(())
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// Processes one input line, writing any output. The String is a buffer to reuse between lines.
|
|
231
|
+
type LineFn<'a> = dyn Fn(&str, &mut String, &mut Vec<u8>) -> io::Result<()> + Sync + 'a;
|
|
232
|
+
|
|
233
|
+
// Runs `process` over every line of the file, writing the output to `out` in input order.
|
|
234
|
+
// Sequentially, the file is streamed. With more than one thread, it is read into memory and
|
|
235
|
+
// divided into blocks that threads claim in turn; the blocks' outputs are then written in order.
|
|
236
|
+
fn run_lines(path: &str, threads: Option<usize>, process: &LineFn, out: &mut dyn Write) -> io::Result<()> {
|
|
237
|
+
let threads = match threads {
|
|
238
|
+
Some(0) => std::thread::available_parallelism().map_or(1, |n| n.get()),
|
|
239
|
+
Some(n) => n,
|
|
240
|
+
None => 1,
|
|
241
|
+
};
|
|
242
|
+
if threads == 1 {
|
|
243
|
+
let (mut seq, mut buf) = (String::new(), Vec::new());
|
|
244
|
+
for_each_line(path, |line| {
|
|
245
|
+
process(line, &mut seq, &mut buf)?;
|
|
246
|
+
if buf.len() >= 1 << 20 {
|
|
247
|
+
out.write_all(&buf)?;
|
|
248
|
+
buf.clear();
|
|
249
|
+
}
|
|
250
|
+
Ok(())
|
|
251
|
+
})?;
|
|
252
|
+
return out.write_all(&buf);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
let text = std::fs::read_to_string(path).map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?;
|
|
256
|
+
let blocks = split_blocks(&text, threads);
|
|
257
|
+
let next_block = AtomicUsize::new(0);
|
|
258
|
+
// One slot per block for its output (or error), filled by a worker and drained in order by this thread.
|
|
259
|
+
let slots = Mutex::new((0..blocks.len()).map(|_| None).collect::<Vec<Option<io::Result<Vec<u8>>>>>());
|
|
260
|
+
let slot_filled = Condvar::new();
|
|
261
|
+
std::thread::scope(|scope| {
|
|
262
|
+
for _ in 0..threads {
|
|
263
|
+
scope.spawn(|| {
|
|
264
|
+
let mut seq = String::new();
|
|
265
|
+
loop {
|
|
266
|
+
let i = next_block.fetch_add(1, Relaxed);
|
|
267
|
+
let Some(block) = blocks.get(i) else { break };
|
|
268
|
+
let mut buf = Vec::with_capacity(block.len() * 5); // Output is several times the size of the input.
|
|
269
|
+
let result = text[block.clone()].lines().try_for_each(|line| process(line, &mut seq, &mut buf));
|
|
270
|
+
slots.lock().unwrap()[i] = Some(result.map(|()| buf));
|
|
271
|
+
slot_filled.notify_one();
|
|
272
|
+
}
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
for i in 0..blocks.len() {
|
|
276
|
+
let mut guard = slots.lock().unwrap();
|
|
277
|
+
while guard[i].is_none() {
|
|
278
|
+
guard = slot_filled.wait(guard).unwrap();
|
|
279
|
+
}
|
|
280
|
+
let result = guard[i].take().unwrap();
|
|
281
|
+
drop(guard);
|
|
282
|
+
out.write_all(&result?)?;
|
|
283
|
+
}
|
|
284
|
+
Ok(())
|
|
285
|
+
})
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// Divides the text into blocks of whole lines. Blocks are small enough that every thread gets several,
|
|
289
|
+
// which balances uneven work, and large enough that handing them out costs nothing.
|
|
290
|
+
fn split_blocks(text: &str, threads: usize) -> Vec<Range<usize>> {
|
|
291
|
+
let target = (text.len() / (4 * threads)).clamp(16 << 10, 4 << 20);
|
|
292
|
+
let mut blocks = Vec::new();
|
|
293
|
+
let mut start = 0;
|
|
294
|
+
while start < text.len() {
|
|
295
|
+
let mut end = (start + target).min(text.len());
|
|
296
|
+
if end < text.len() {
|
|
297
|
+
end = text[end..].find('\n').map_or(text.len(), |i| end + i + 1);
|
|
298
|
+
}
|
|
299
|
+
blocks.push(start..end);
|
|
300
|
+
start = end;
|
|
301
|
+
}
|
|
302
|
+
blocks
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// Calls `f` with each line of the file, stripped of its line ending. One String is reused for all lines.
|
|
306
|
+
fn for_each_line(path: &str, mut f: impl FnMut(&str) -> io::Result<()>) -> io::Result<()> {
|
|
307
|
+
let file = File::open(path).map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?;
|
|
308
|
+
let mut reader = BufReader::with_capacity(1 << 20, file);
|
|
309
|
+
let mut line = String::new();
|
|
310
|
+
loop {
|
|
311
|
+
line.clear();
|
|
312
|
+
if reader.read_line(&mut line)? == 0 {
|
|
313
|
+
return Ok(());
|
|
314
|
+
}
|
|
315
|
+
f(line.trim_end_matches(['\n', '\r']))?;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
fn is_skippable(line: &str, bed: bool) -> bool {
|
|
320
|
+
let t = line.trim();
|
|
321
|
+
t.is_empty() || t.starts_with('#') || (bed && (t.starts_with("track") || t.starts_with("browser")))
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// Splits "name" or "name:start-end" (zero-based, half-open). A colon that is not followed by
|
|
325
|
+
// a valid range is treated as part of the name.
|
|
326
|
+
fn parse_spec(spec: &str) -> (&str, Option<(usize, usize)>) {
|
|
327
|
+
if let Some((name, range)) = spec.rsplit_once(':')
|
|
328
|
+
&& let Some((s, e)) = range.split_once('-')
|
|
329
|
+
&& let (Ok(s), Ok(e)) = (s.replace(',', "").parse(), e.replace(',', "").parse())
|
|
330
|
+
{
|
|
331
|
+
return (name, Some((s, e)));
|
|
332
|
+
}
|
|
333
|
+
(spec, None)
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// Adds the input file to an error from the library.
|
|
337
|
+
fn in_file(e: io::Error, file: &str) -> io::Error {
|
|
338
|
+
io::Error::new(e.kind(), format!("{file}: {e}"))
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
fn check_range(name: &str, start: usize, end: usize, len: usize) -> io::Result<()> {
|
|
342
|
+
if start >= end {
|
|
343
|
+
return err(format!("{name}: start ({start}) must be less than end ({end})"));
|
|
344
|
+
}
|
|
345
|
+
if end > len {
|
|
346
|
+
return err(format!("{name}: end ({end}) is past the end of the sequence (length {len})"));
|
|
347
|
+
}
|
|
348
|
+
Ok(())
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
// Writes a record: ">header", then the sequence wrapped at LINE_WIDTH columns.
|
|
352
|
+
fn write_record(out: &mut (impl Write + ?Sized), header: Arguments, seq: &str) -> io::Result<()> {
|
|
353
|
+
writeln!(out, ">{header}")?;
|
|
354
|
+
for line in seq.as_bytes().chunks(LINE_WIDTH) {
|
|
355
|
+
out.write_all(line)?;
|
|
356
|
+
out.write_all(b"\n")?;
|
|
357
|
+
}
|
|
358
|
+
Ok(())
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Writes the record for `name` or `name:start-end`. `start`/`end` (from -start/-end) apply only if the
|
|
362
|
+
// spec has no range. Like twoBitToFa, they select a range only when end > start; otherwise the whole
|
|
363
|
+
// sequence is written.
|
|
364
|
+
fn write_spec(
|
|
365
|
+
tbr: &TwobitReader,
|
|
366
|
+
file: &str,
|
|
367
|
+
spec: &str,
|
|
368
|
+
start: Option<usize>,
|
|
369
|
+
end: Option<usize>,
|
|
370
|
+
seq: &mut String,
|
|
371
|
+
out: &mut impl Write,
|
|
372
|
+
) -> io::Result<()> {
|
|
373
|
+
let (name, range) = parse_spec(spec);
|
|
374
|
+
if !tbr.contains_name(name) {
|
|
375
|
+
return err(format!("sequence {name} not found in {file}"));
|
|
376
|
+
}
|
|
377
|
+
let len = tbr.seq_len(name);
|
|
378
|
+
let (start, end) = (start.unwrap_or(0), end.unwrap_or(0));
|
|
379
|
+
let range = range.or((end > start).then_some((start, end)));
|
|
380
|
+
let (s, e) = range.unwrap_or((0, len));
|
|
381
|
+
check_range(name, s, e, len)?;
|
|
382
|
+
tbr.get_into(name, s, e, seq);
|
|
383
|
+
match range {
|
|
384
|
+
Some(_) => write_record(out, format_args!("{name}:{s}-{e}"), seq),
|
|
385
|
+
None => write_record(out, format_args!("{name}"), seq),
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// Writes the record for one BED line (BED3 to BED12). BED12 blocks are joined, and minus-strand
|
|
390
|
+
// records are reverse-complemented.
|
|
391
|
+
fn write_bed_record(
|
|
392
|
+
tbr: &TwobitReader,
|
|
393
|
+
file: &str,
|
|
394
|
+
line: &str,
|
|
395
|
+
bed_pos: bool,
|
|
396
|
+
seq: &mut String,
|
|
397
|
+
out: &mut impl Write,
|
|
398
|
+
) -> io::Result<()> {
|
|
399
|
+
let bad = |what: &str| err(format!("bad BED line ({what}): {line}"));
|
|
400
|
+
let mut cols = line.split_ascii_whitespace();
|
|
401
|
+
let mut next = || cols.next();
|
|
402
|
+
let (Some(chrom), Some(start), Some(end)) = (next(), next(), next()) else {
|
|
403
|
+
return bad("need at least 3 columns");
|
|
404
|
+
};
|
|
405
|
+
let (Ok(start), Ok(end)) = (start.parse::<usize>(), end.parse::<usize>()) else {
|
|
406
|
+
return bad("non-numeric start or end");
|
|
407
|
+
};
|
|
408
|
+
let (name, _score, strand) = (next(), next(), next());
|
|
409
|
+
let (_thick_start, _thick_end, _rgb, count, sizes, starts) = (next(), next(), next(), next(), next(), next());
|
|
410
|
+
|
|
411
|
+
if start >= end {
|
|
412
|
+
return err(format!("{chrom}: start ({start}) must be less than end ({end})"));
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
if let (Some(count), Some(sizes), Some(starts)) = (count, sizes, starts) {
|
|
416
|
+
let list = |s: &str| s.split(',').filter(|x| !x.is_empty()).map(|x| x.parse::<usize>().ok()).collect();
|
|
417
|
+
let (Ok(count), Some::<Vec<_>>(sizes), Some::<Vec<_>>(starts)) =
|
|
418
|
+
(count.parse::<usize>(), list(sizes), list(starts))
|
|
419
|
+
else {
|
|
420
|
+
return bad("blockCount, blockSizes or blockStarts");
|
|
421
|
+
};
|
|
422
|
+
if sizes.len() != count || starts.len() != count {
|
|
423
|
+
return bad("blockCount does not match blockSizes/blockStarts");
|
|
424
|
+
}
|
|
425
|
+
let ranges: Vec<_> = starts.iter().zip(&sizes).map(|(b, size)| (start + b, start + b + size)).collect();
|
|
426
|
+
if ranges.iter().any(|&(s, e)| s == e || e > end) {
|
|
427
|
+
return bad("block outside the feature");
|
|
428
|
+
}
|
|
429
|
+
tbr.try_get_into(chrom, end, end, seq).map_err(|e| in_file(e, file))?; // Checks the name and end; extracts nothing.
|
|
430
|
+
*seq = tbr.concat(chrom, &ranges);
|
|
431
|
+
} else {
|
|
432
|
+
tbr.try_get_into(chrom, start, end, seq).map_err(|e| in_file(e, file))?;
|
|
433
|
+
}
|
|
434
|
+
if strand == Some("-") {
|
|
435
|
+
*seq = reverse_complement(std::mem::take(seq)); // In place, so no reallocation.
|
|
436
|
+
}
|
|
437
|
+
match name {
|
|
438
|
+
Some(name) if !bed_pos => write_record(out, format_args!("{name}"), seq),
|
|
439
|
+
_ => write_record(out, format_args!("{chrom}:{start}-{end}"), seq),
|
|
440
|
+
}
|
|
441
|
+
}
|
|
@@ -2,22 +2,14 @@
|
|
|
2
2
|
#![deny(missing_docs)]
|
|
3
3
|
#![deny(unreachable_pub)]
|
|
4
4
|
#![warn(rust_2018_idioms)]
|
|
5
|
-
//!
|
|
6
5
|
//! This crate provides fast DNA sequence extraction from 2bit files, a
|
|
7
6
|
//! [standard format](http://genome.ucsc.edu/FAQ/FAQformat.html#format7) in bioinformatics.
|
|
8
7
|
//!
|
|
9
|
-
//! The motivation for `twobitreader` is speed; see benchmarks below.
|
|
10
|
-
//! It is also available as a Python package named `twobitreader-rs`.
|
|
11
|
-
//!
|
|
12
8
|
//! The focus is raw reading from 2bit, but fast concatenation and reverse-complement methods
|
|
13
9
|
//! are also provided to make higher-level use cases easier.
|
|
14
10
|
//!
|
|
15
|
-
//!
|
|
16
|
-
//!
|
|
17
|
-
//! 
|
|
18
|
-
//! 
|
|
19
|
-
//! [](https://anaconda.org/conda-forge/twobitreader-rs)
|
|
20
|
-
//! [](https://pypi.org/project/twobitreader-rs/)
|
|
11
|
+
//! The crate is also available as a Python package (`twobitreader-rs`) or as a
|
|
12
|
+
//! terminal command replacing `twoBitToFa`.
|
|
21
13
|
//!
|
|
22
14
|
//! # Examples
|
|
23
15
|
//!
|
|
@@ -106,6 +98,16 @@
|
|
|
106
98
|
//! # Ok::<(), io::Error>(())
|
|
107
99
|
//! ```
|
|
108
100
|
//!
|
|
101
|
+
//! # Command line
|
|
102
|
+
//!
|
|
103
|
+
//! The `cli` feature builds a `twobitreader` command that is a drop-in replacement for the core features
|
|
104
|
+
//! of UCSC `twoBitToFa`:
|
|
105
|
+
//! ```console
|
|
106
|
+
//! cargo install twobitreader --features cli
|
|
107
|
+
//! twobitreader -seq=chr1 -start=10000 -end=10005 hg38.2bit stdout
|
|
108
|
+
//! twobitreader -bed=exons.bed -threads=16 hg38.2bit out.fa
|
|
109
|
+
//! ```
|
|
110
|
+
//!
|
|
109
111
|
//! # Benchmarks
|
|
110
112
|
//!
|
|
111
113
|
//! Two tasks were benchmarked:
|
|
@@ -117,13 +119,14 @@
|
|
|
117
119
|
//! - **cold** runs represent a first run of a genomics pipeline, bound by disk speed;
|
|
118
120
|
//! - **prefetch** runs are cold but with a `prefetch` call preceding extraction.
|
|
119
121
|
//!
|
|
120
|
-
//! The table below shows running times in milliseconds. Experimental details are `BENCH.md`.
|
|
121
|
-
//! This crate provides
|
|
122
|
+
//! The table below shows running times in milliseconds. Experimental details are in `BENCH.md`.
|
|
123
|
+
//! This crate provides the top three rows: *twobitreader*, *twobitreader-rs*, *twobitreader-cli*.
|
|
122
124
|
//!
|
|
123
125
|
//! | EXONS | 1-thread / hot | 1-thread / cold | 1-thread / prefetch | 16-thread / hot | 16-thread / cold |
|
|
124
126
|
//! |---|---:|---:|---:|---:|---:|
|
|
125
127
|
//! | **twobitreader** (rs) | 90 | 1,700 | 180 | 10 | 190 |
|
|
126
|
-
//! | **
|
|
128
|
+
//! | **twobitreader-rs** (rs, py) | 100 | 1,900 | 190 | *27 | *210 |
|
|
129
|
+
//! | **twobitreader-cli** (rs) | 150 | 2,200 | 280 | 55 | 260 |
|
|
127
130
|
//! | **py2bit** (c, py) | 220 | 2,800 | n/a | n/a | n/a |
|
|
128
131
|
//! | **GenomeKit** (cpp, py) | 340 | 2,400 | n/a | n/a | n/a |
|
|
129
132
|
//! | **twobit** (rs) | 390 | 3,500 | n/a | n/a | n/a |
|
|
@@ -134,7 +137,8 @@
|
|
|
134
137
|
//! | TRANSCRIPTS | 1-thread / hot | 1-thread / cold | 1-thread / prefetch | 16-thread / hot | 16-thread / cold |
|
|
135
138
|
//! |---|---:|---:|---:|---:|---:|
|
|
136
139
|
//! | **twobitreader** (rs) | 140 | 1,700 | 230 | 13 | 190 |
|
|
137
|
-
//! | **
|
|
140
|
+
//! | **twobitreader-rs** (rs, py) | 180 | 2,100 | 310 | *27 | *220 |
|
|
141
|
+
//! | **twobitreader-cli** (rs) | 250 | 2,300 | 340 | 75 | 300 |
|
|
138
142
|
//! | **py2bit** (c, py) | 490 | 3,600 | n/a | n/a | n/a |
|
|
139
143
|
//! | **GenomeKit** (cpp, py) | 720 | 2,900 | n/a | n/a | n/a |
|
|
140
144
|
//! | **twobit** (rs) | 930 | 1,900 | n/a | n/a | n/a |
|
|
@@ -154,13 +158,15 @@
|
|
|
154
158
|
//!
|
|
155
159
|
//! # Use of AI
|
|
156
160
|
//!
|
|
157
|
-
//!
|
|
158
|
-
//!
|
|
159
|
-
//!
|
|
161
|
+
//! The Rust library was written primarily by hand.
|
|
162
|
+
//!
|
|
163
|
+
//! Claude was used to write the CLI and its tests+benchmark, to generated the OS-specific
|
|
164
|
+
//! prefetch loops, to improve handling of corrupt or malicious files, to document the
|
|
165
|
+
//! Python bindings and to write their tests+benchmark, to improved error checking
|
|
166
|
+
//! and propagation to Python more broadly, and to generated the CI configurations.
|
|
160
167
|
|
|
161
168
|
// Standard library
|
|
162
169
|
use std::borrow::Borrow;
|
|
163
|
-
use std::collections::HashMap;
|
|
164
170
|
use std::fs::File;
|
|
165
171
|
use std::io::{self, Cursor, ErrorKind::InvalidData, Read};
|
|
166
172
|
use std::iter::zip;
|
|
@@ -175,6 +181,8 @@ mod decode;
|
|
|
175
181
|
use decode::{NUCS_PER_U8, decode};
|
|
176
182
|
mod prefetch;
|
|
177
183
|
use prefetch::PrefetchBatcher;
|
|
184
|
+
mod seq_index_map;
|
|
185
|
+
use seq_index_map::SeqIndexMap;
|
|
178
186
|
#[cfg(feature = "python")]
|
|
179
187
|
mod python;
|
|
180
188
|
|
|
@@ -199,12 +207,12 @@ enum Endianness {
|
|
|
199
207
|
///
|
|
200
208
|
#[derive(Debug)]
|
|
201
209
|
pub struct TwobitReader {
|
|
202
|
-
file: File,
|
|
203
|
-
mmap: Mmap,
|
|
204
|
-
masked: bool,
|
|
205
|
-
endianness: Endianness,
|
|
206
|
-
seqs: Vec<TwobitSequence>,
|
|
207
|
-
seq_by_name:
|
|
210
|
+
file: File, // Open 2bit file; kept for issuing prefetch hints.
|
|
211
|
+
mmap: Mmap, // Memory map of entire 2bit file.
|
|
212
|
+
masked: bool, // Whether to apply the lowercase mask.
|
|
213
|
+
endianness: Endianness, // Whether the 2bit file is big-endian.
|
|
214
|
+
seqs: Vec<TwobitSequence>, // Sequence data, in same order as in the file.
|
|
215
|
+
seq_by_name: SeqIndexMap, // Lookup sequence index (seqs[index]) by name.
|
|
208
216
|
}
|
|
209
217
|
|
|
210
218
|
/// A specific sequence record within the 2bit file.
|
|
@@ -284,7 +292,7 @@ impl TwobitReader {
|
|
|
284
292
|
let seqs = read_seqs(&mmap, endianness)?;
|
|
285
293
|
|
|
286
294
|
// Allow fast lookup of sequences by name
|
|
287
|
-
let seq_by_name =
|
|
295
|
+
let seq_by_name = SeqIndexMap::from_iter(seqs.iter().enumerate().map(|(i, seq)| (seq.name.clone(), i)));
|
|
288
296
|
if seq_by_name.len() != seqs.len() {
|
|
289
297
|
return Err(io::Error::new(InvalidData, "duplicate sequence name detected."));
|
|
290
298
|
}
|
|
@@ -367,7 +375,40 @@ impl TwobitReader {
|
|
|
367
375
|
// Check range before trying to do any might-panic arithmetic with (start, end)
|
|
368
376
|
let seq = self.get_seq_data_by_name(name);
|
|
369
377
|
check_range(seq, start, end);
|
|
378
|
+
self.get_checked_into(seq, start, end, dst);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/// A version of [`get_into`](Self::get_into) that returns an error if the sequence name was
|
|
382
|
+
/// not found or the range was invalid. The error message names the sequence.
|
|
383
|
+
///
|
|
384
|
+
/// This function facilitates better error reporting while still using only a single hash lookup
|
|
385
|
+
/// of the sequence name.
|
|
386
|
+
///
|
|
387
|
+
/// # Panics
|
|
388
|
+
///
|
|
389
|
+
/// Panics only if an IO error occurs due to a malformed file.
|
|
390
|
+
pub fn try_get_into<N: AsRef<str>>(&self, name: N, start: usize, end: usize, dst: &mut String) -> io::Result<()> {
|
|
391
|
+
let name = name.as_ref();
|
|
392
|
+
let seq = self
|
|
393
|
+
.try_get_seq_data_by_name(name)
|
|
394
|
+
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("sequence {name} not found")))?;
|
|
395
|
+
|
|
396
|
+
// Check the range and return an error
|
|
397
|
+
let invalid = |msg: String| io::Error::new(io::ErrorKind::InvalidInput, format!("{name}: {msg}"));
|
|
398
|
+
if start > end {
|
|
399
|
+
return Err(invalid(format!("invalid range (start {start} > end {end})")));
|
|
400
|
+
}
|
|
401
|
+
if end > seq.dna_len {
|
|
402
|
+
return Err(invalid(format!("end ({end}) is past the end of the sequence (length {})", seq.dna_len)));
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
// Decode the checked range
|
|
406
|
+
self.get_checked_into(seq, start, end, dst);
|
|
407
|
+
Ok(())
|
|
408
|
+
}
|
|
370
409
|
|
|
410
|
+
// Implements get_into for a range that has already been checked.
|
|
411
|
+
fn get_checked_into(&self, seq: &TwobitSequenceData, start: usize, end: usize, dst: &mut String) {
|
|
371
412
|
// Prepare an empty buffer with sufficient capacity.
|
|
372
413
|
// SAFETY: clearing dst ensures that its visible portion remains valid utf8
|
|
373
414
|
// until decode_and_append lengthens it.
|
|
@@ -910,7 +951,7 @@ fn read_seqs_endian<B: ByteOrder>(mmap: &Mmap) -> io::Result<Vec<TwobitSequence>
|
|
|
910
951
|
// Get the offset of the data arrays for this sequence
|
|
911
952
|
let data_offset = cursor.read_u32::<B>()? as u64;
|
|
912
953
|
let curr_offset = cursor.position();
|
|
913
|
-
if data_offset
|
|
954
|
+
if data_offset < curr_offset {
|
|
914
955
|
return Err(io::Error::new(InvalidData, "invalid data offset; file may be malformed."));
|
|
915
956
|
}
|
|
916
957
|
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// A specialized stand-in for `HashMap<String, usize>`, mapping sequence names to indices into
|
|
2
|
+
// `TwobitReader::seqs`.
|
|
3
|
+
//
|
|
4
|
+
// Human genomes are accessed almost entirely by the primary names chr1..chr22, chrX, chrY and chrM, so
|
|
5
|
+
// those names are mapped to an array slot by inspecting the characters, with no hashing or comparison.
|
|
6
|
+
// Every other name (alternate contigs such as chr1_KI270706v1_random, or non-human names) falls back to
|
|
7
|
+
// a HashMap.
|
|
8
|
+
|
|
9
|
+
use std::collections::HashMap;
|
|
10
|
+
|
|
11
|
+
// The slots are the chromosome numbers 1..=22 (slot 0 is unused), then chrX, chrY and chrM.
|
|
12
|
+
const SLOT_X: usize = 23;
|
|
13
|
+
const SLOT_Y: usize = 24;
|
|
14
|
+
const SLOT_M: usize = 25;
|
|
15
|
+
const NUM_SLOTS: usize = 26;
|
|
16
|
+
|
|
17
|
+
// Marks a slot that no sequence maps to.
|
|
18
|
+
const EMPTY: usize = usize::MAX;
|
|
19
|
+
|
|
20
|
+
// Returns the slot for a primary sequence name, or None for any other name.
|
|
21
|
+
// Each name maps to its own slot, and no two names share one.
|
|
22
|
+
fn direct_slot(name: &str) -> Option<usize> {
|
|
23
|
+
match name.strip_prefix("chr")?.as_bytes() {
|
|
24
|
+
[digit @ b'1'..=b'9'] => Some(usize::from(digit - b'0')), // chr1..chr9
|
|
25
|
+
[tens @ b'1'..=b'2', ones @ b'0'..=b'9'] => {
|
|
26
|
+
let number = usize::from(tens - b'0') * 10 + usize::from(ones - b'0');
|
|
27
|
+
(number <= 22).then_some(number) // chr10..chr22, but not chr23..chr29
|
|
28
|
+
}
|
|
29
|
+
b"X" => Some(SLOT_X),
|
|
30
|
+
b"Y" => Some(SLOT_Y),
|
|
31
|
+
b"M" => Some(SLOT_M),
|
|
32
|
+
_ => None,
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
#[derive(Debug)]
|
|
37
|
+
pub(crate) struct SeqIndexMap {
|
|
38
|
+
direct: Vec<usize>, // direct[direct_slot(name)] is the sequence index, or EMPTY.
|
|
39
|
+
other: HashMap<String, usize>, // The sequence index for each name that has no direct slot.
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
impl SeqIndexMap {
|
|
43
|
+
// Returns the sequence index for the name, if there is one.
|
|
44
|
+
pub(crate) fn get(&self, name: &str) -> Option<&usize> {
|
|
45
|
+
match direct_slot(name) {
|
|
46
|
+
Some(slot) => Some(&self.direct[slot]).filter(|&&index| index != EMPTY),
|
|
47
|
+
None => self.other.get(name),
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
pub(crate) fn contains_key(&self, name: &str) -> bool {
|
|
52
|
+
self.get(name).is_some()
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Returns the number of distinct names.
|
|
56
|
+
pub(crate) fn len(&self) -> usize {
|
|
57
|
+
self.direct.iter().filter(|&&index| index != EMPTY).count() + self.other.len()
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// As with HashMap, when a name occurs more than once the last index wins and len() counts it once.
|
|
62
|
+
impl FromIterator<(String, usize)> for SeqIndexMap {
|
|
63
|
+
fn from_iter<I: IntoIterator<Item = (String, usize)>>(iter: I) -> Self {
|
|
64
|
+
let mut map = Self { direct: vec![EMPTY; NUM_SLOTS], other: HashMap::new() };
|
|
65
|
+
for (name, index) in iter {
|
|
66
|
+
match direct_slot(&name) {
|
|
67
|
+
Some(slot) => map.direct[slot] = index,
|
|
68
|
+
None => {
|
|
69
|
+
map.other.insert(name, index);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
map
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
#[cfg(test)]
|
|
78
|
+
mod tests {
|
|
79
|
+
use super::*;
|
|
80
|
+
|
|
81
|
+
fn primary_names() -> Vec<String> {
|
|
82
|
+
let numbered = (1..=22).map(|n| format!("chr{n}"));
|
|
83
|
+
numbered.chain(["chrX", "chrY", "chrM"].map(String::from)).collect()
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
#[test]
|
|
87
|
+
fn primary_names_have_distinct_slots() {
|
|
88
|
+
let slots: Vec<_> = primary_names().iter().map(|name| direct_slot(name).unwrap()).collect();
|
|
89
|
+
assert_eq!(slots, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
#[test]
|
|
93
|
+
fn other_names_have_no_slot() {
|
|
94
|
+
let names = [
|
|
95
|
+
"",
|
|
96
|
+
"chr",
|
|
97
|
+
"chr0",
|
|
98
|
+
"chr00",
|
|
99
|
+
"chr01",
|
|
100
|
+
"chr23",
|
|
101
|
+
"chr29",
|
|
102
|
+
"chr30",
|
|
103
|
+
"chr100",
|
|
104
|
+
"chrx",
|
|
105
|
+
"chry",
|
|
106
|
+
"chrm",
|
|
107
|
+
"chrXY",
|
|
108
|
+
"chrMT",
|
|
109
|
+
"Chr1",
|
|
110
|
+
"CHR1",
|
|
111
|
+
"1",
|
|
112
|
+
"X",
|
|
113
|
+
"chr1_KI270706v1_random",
|
|
114
|
+
"chrUn_GL000195v1",
|
|
115
|
+
"chrEBV",
|
|
116
|
+
"seq_abc",
|
|
117
|
+
];
|
|
118
|
+
for name in names {
|
|
119
|
+
assert_eq!(direct_slot(name), None, "{name:?}");
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Behaves like a HashMap, whether the names have direct slots or not.
|
|
124
|
+
#[test]
|
|
125
|
+
fn matches_hashmap() {
|
|
126
|
+
let mut names = primary_names();
|
|
127
|
+
names.extend(
|
|
128
|
+
["chr1_KI270706v1_random", "chrUn_GL000195v1", "chrEBV", "seq_abc", "chr23", "chr01", ""].map(String::from),
|
|
129
|
+
);
|
|
130
|
+
let pairs: Vec<_> = names.iter().cloned().zip(0..).collect();
|
|
131
|
+
let expected: HashMap<String, usize> = pairs.iter().cloned().collect();
|
|
132
|
+
let map: SeqIndexMap = pairs.into_iter().collect();
|
|
133
|
+
assert_eq!(map.len(), expected.len());
|
|
134
|
+
for name in names.iter().map(String::as_str).chain(["chr24", "chr", "chrZ", "nope", "Chr1"]) {
|
|
135
|
+
assert_eq!(map.get(name), expected.get(name), "{name:?}");
|
|
136
|
+
assert_eq!(map.contains_key(name), expected.contains_key(name), "{name:?}");
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
#[test]
|
|
141
|
+
fn duplicates_count_once() {
|
|
142
|
+
let map: SeqIndexMap =
|
|
143
|
+
[("chr1", 0), ("chr1", 7), ("seq", 1), ("seq", 2)].map(|(n, i)| (n.to_string(), i)).into_iter().collect();
|
|
144
|
+
assert_eq!(map.len(), 2);
|
|
145
|
+
assert_eq!((map.get("chr1"), map.get("seq")), (Some(&7), Some(&2)));
|
|
146
|
+
}
|
|
147
|
+
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|