twobitreader-rs 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- twobitreader_rs-0.2.1/CHANGELOG.md +72 -0
- twobitreader_rs-0.2.1/Cargo.lock +297 -0
- twobitreader_rs-0.2.1/Cargo.toml +42 -0
- twobitreader_rs-0.2.1/LICENSE-APACHE +13 -0
- twobitreader_rs-0.2.1/LICENSE-MIT +25 -0
- twobitreader_rs-0.2.1/PKG-INFO +28 -0
- twobitreader_rs-0.2.1/README.md +132 -0
- twobitreader_rs-0.2.1/custom.css +28 -0
- twobitreader_rs-0.2.1/py.typed +0 -0
- twobitreader_rs-0.2.1/pyproject.toml +40 -0
- twobitreader_rs-0.2.1/src/decode.rs +149 -0
- twobitreader_rs-0.2.1/src/lib.rs +1139 -0
- twobitreader_rs-0.2.1/src/prefetch.rs +130 -0
- twobitreader_rs-0.2.1/src/python.rs +672 -0
- twobitreader_rs-0.2.1/twobitreader_rs.pyi +65 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here.
|
|
4
|
+
|
|
5
|
+
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this
|
|
6
|
+
project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). The Rust
|
|
7
|
+
crate `twobitreader` and the Python package `twobitreader_rs` share a version number.
|
|
8
|
+
|
|
9
|
+
## [Unreleased]
|
|
10
|
+
|
|
11
|
+
## [0.2.1] - 2026-10-01
|
|
12
|
+
|
|
13
|
+
First release of `twobitreader_rs` on PyPI. There are no library changes; 0.2.0 reached
|
|
14
|
+
crates.io before the wheel-building workflow was in place, so it was never published for
|
|
15
|
+
Python.
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- Pre-built wheels on PyPI for CPython 3.9 through 3.14, including the free-threaded
|
|
20
|
+
3.14t build, across Linux (glibc and musl), macOS and Windows, each on x86_64 and
|
|
21
|
+
aarch64. A source distribution is published alongside them for any platform or
|
|
22
|
+
interpreter not covered, PyPy included.
|
|
23
|
+
- A release workflow that builds and publishes those wheels when a version tag is pushed.
|
|
24
|
+
- A statement on Use of AI.
|
|
25
|
+
|
|
26
|
+
## [0.2.0] - 2026-10-01
|
|
27
|
+
|
|
28
|
+
`twobitreader` is now available from Python as **`twobitreader_rs`**, built with PyO3 and
|
|
29
|
+
published as a wheel. The Python API mirrors the Rust one, with speed close to pure Rust.
|
|
30
|
+
|
|
31
|
+
### Added
|
|
32
|
+
|
|
33
|
+
- **Python bindings** (`twobitreader_rs`), exposing `TwobitReader` and `reverse_complement`.
|
|
34
|
+
Available for CPython 3.9+ on Linux, macOS and Windows.
|
|
35
|
+
- **Free-threaded CPython support.** The extension is marked `gil_used = false` and the
|
|
36
|
+
reader is immutable and shareable. A thread pool over batches scales across cores on a
|
|
37
|
+
free-threaded build.
|
|
38
|
+
- **Type stubs** (`py.typed` and `__init__.pyi`), so editors and type checkers see full
|
|
39
|
+
signatures for the compiled module.
|
|
40
|
+
- **`prefetch_inclusive`**, the 1-based counterpart of `prefetch`, which was missing.
|
|
41
|
+
|
|
42
|
+
### Changed
|
|
43
|
+
|
|
44
|
+
- CI moved from CircleCI to GitHub Actions, now also building and testing the Python wheel.
|
|
45
|
+
|
|
46
|
+
### Security
|
|
47
|
+
|
|
48
|
+
Hardened against malformed 2bit files; the issues were unbounded allocations triggered by
|
|
49
|
+
a crafted file, not memory-safety defects.
|
|
50
|
+
|
|
51
|
+
### Known issues
|
|
52
|
+
|
|
53
|
+
- Reading a 2bit file that another process modifies concurrently yields undefined behaviour.
|
|
54
|
+
|
|
55
|
+
## [0.1.1] - 2026-09-25
|
|
56
|
+
|
|
57
|
+
### Changed
|
|
58
|
+
|
|
59
|
+
- Documentation fixes and updated crate keywords.
|
|
60
|
+
|
|
61
|
+
## [0.1.0] - 2026-09-24
|
|
62
|
+
|
|
63
|
+
Initial release of the `twobitreader` crate: sequence extraction with `get`, batched
|
|
64
|
+
extraction with `get_batch`, exon concatenation with `concat`, reverse complement, and
|
|
65
|
+
`prefetch` for cold files. Sequence records are parsed lazily so that opening a large file
|
|
66
|
+
stays cheap. No git tag was made for this version.
|
|
67
|
+
|
|
68
|
+
[Unreleased]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.2.1...HEAD
|
|
69
|
+
[0.2.1]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.2.0...v0.2.1
|
|
70
|
+
[0.2.0]: https://github.com/andrewdelong/twobitreader-rust/compare/v0.1.1...v0.2.0
|
|
71
|
+
[0.1.1]: https://github.com/andrewdelong/twobitreader-rust/releases/tag/v0.1.1
|
|
72
|
+
[0.1.0]: https://crates.io/crates/twobitreader/0.1.0
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "adler2"
|
|
7
|
+
version = "2.0.1"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
|
10
|
+
|
|
11
|
+
[[package]]
|
|
12
|
+
name = "byteorder"
|
|
13
|
+
version = "1.5.0"
|
|
14
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
15
|
+
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
|
16
|
+
|
|
17
|
+
[[package]]
|
|
18
|
+
name = "cfg-if"
|
|
19
|
+
version = "1.0.5"
|
|
20
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
21
|
+
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
|
22
|
+
|
|
23
|
+
[[package]]
|
|
24
|
+
name = "crc32fast"
|
|
25
|
+
version = "1.5.2"
|
|
26
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
27
|
+
checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
|
|
28
|
+
dependencies = [
|
|
29
|
+
"cfg-if",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[[package]]
|
|
33
|
+
name = "crossbeam-deque"
|
|
34
|
+
version = "0.8.8"
|
|
35
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
36
|
+
checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
|
|
37
|
+
dependencies = [
|
|
38
|
+
"crossbeam-epoch",
|
|
39
|
+
"crossbeam-utils",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
[[package]]
|
|
43
|
+
name = "crossbeam-epoch"
|
|
44
|
+
version = "0.9.21"
|
|
45
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
46
|
+
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
|
|
47
|
+
dependencies = [
|
|
48
|
+
"crossbeam-utils",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[[package]]
|
|
52
|
+
name = "crossbeam-utils"
|
|
53
|
+
version = "0.8.23"
|
|
54
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
55
|
+
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
|
56
|
+
|
|
57
|
+
[[package]]
|
|
58
|
+
name = "either"
|
|
59
|
+
version = "1.18.0"
|
|
60
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
61
|
+
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
|
|
62
|
+
|
|
63
|
+
[[package]]
|
|
64
|
+
name = "flate2"
|
|
65
|
+
version = "1.1.10"
|
|
66
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
67
|
+
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
|
68
|
+
dependencies = [
|
|
69
|
+
"crc32fast",
|
|
70
|
+
"miniz_oxide",
|
|
71
|
+
"zlib-rs",
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
[[package]]
|
|
75
|
+
name = "heck"
|
|
76
|
+
version = "0.5.0"
|
|
77
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
78
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
79
|
+
|
|
80
|
+
[[package]]
|
|
81
|
+
name = "itertools"
|
|
82
|
+
version = "0.15.0"
|
|
83
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
84
|
+
checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
|
|
85
|
+
dependencies = [
|
|
86
|
+
"either",
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
[[package]]
|
|
90
|
+
name = "libc"
|
|
91
|
+
version = "0.2.189"
|
|
92
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
93
|
+
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
|
94
|
+
|
|
95
|
+
[[package]]
|
|
96
|
+
name = "memmap2"
|
|
97
|
+
version = "0.9.11"
|
|
98
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
99
|
+
checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
|
|
100
|
+
dependencies = [
|
|
101
|
+
"libc",
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
[[package]]
|
|
105
|
+
name = "miniz_oxide"
|
|
106
|
+
version = "0.9.1"
|
|
107
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
108
|
+
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
|
109
|
+
dependencies = [
|
|
110
|
+
"adler2",
|
|
111
|
+
"simd-adler32",
|
|
112
|
+
]
|
|
113
|
+
|
|
114
|
+
[[package]]
|
|
115
|
+
name = "once_cell"
|
|
116
|
+
version = "1.21.4"
|
|
117
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
118
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
119
|
+
|
|
120
|
+
[[package]]
|
|
121
|
+
name = "portable-atomic"
|
|
122
|
+
version = "1.15.0"
|
|
123
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
124
|
+
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
125
|
+
|
|
126
|
+
[[package]]
|
|
127
|
+
name = "proc-macro2"
|
|
128
|
+
version = "1.0.107"
|
|
129
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
130
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
131
|
+
dependencies = [
|
|
132
|
+
"unicode-ident",
|
|
133
|
+
]
|
|
134
|
+
|
|
135
|
+
[[package]]
|
|
136
|
+
name = "pyo3"
|
|
137
|
+
version = "0.29.3"
|
|
138
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
139
|
+
checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655"
|
|
140
|
+
dependencies = [
|
|
141
|
+
"libc",
|
|
142
|
+
"once_cell",
|
|
143
|
+
"portable-atomic",
|
|
144
|
+
"pyo3-build-config",
|
|
145
|
+
"pyo3-ffi",
|
|
146
|
+
"pyo3-macros",
|
|
147
|
+
]
|
|
148
|
+
|
|
149
|
+
[[package]]
|
|
150
|
+
name = "pyo3-build-config"
|
|
151
|
+
version = "0.29.3"
|
|
152
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
153
|
+
checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb"
|
|
154
|
+
dependencies = [
|
|
155
|
+
"target-lexicon",
|
|
156
|
+
]
|
|
157
|
+
|
|
158
|
+
[[package]]
|
|
159
|
+
name = "pyo3-ffi"
|
|
160
|
+
version = "0.29.3"
|
|
161
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
162
|
+
checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330"
|
|
163
|
+
dependencies = [
|
|
164
|
+
"libc",
|
|
165
|
+
"pyo3-build-config",
|
|
166
|
+
]
|
|
167
|
+
|
|
168
|
+
[[package]]
|
|
169
|
+
name = "pyo3-macros"
|
|
170
|
+
version = "0.29.3"
|
|
171
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
172
|
+
checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6"
|
|
173
|
+
dependencies = [
|
|
174
|
+
"proc-macro2",
|
|
175
|
+
"pyo3-macros-backend",
|
|
176
|
+
"quote",
|
|
177
|
+
"syn",
|
|
178
|
+
]
|
|
179
|
+
|
|
180
|
+
[[package]]
|
|
181
|
+
name = "pyo3-macros-backend"
|
|
182
|
+
version = "0.29.3"
|
|
183
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
184
|
+
checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3"
|
|
185
|
+
dependencies = [
|
|
186
|
+
"heck",
|
|
187
|
+
"proc-macro2",
|
|
188
|
+
"quote",
|
|
189
|
+
"syn",
|
|
190
|
+
]
|
|
191
|
+
|
|
192
|
+
[[package]]
|
|
193
|
+
name = "quote"
|
|
194
|
+
version = "1.0.47"
|
|
195
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
196
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
197
|
+
dependencies = [
|
|
198
|
+
"proc-macro2",
|
|
199
|
+
]
|
|
200
|
+
|
|
201
|
+
[[package]]
|
|
202
|
+
name = "rayon"
|
|
203
|
+
version = "1.12.0"
|
|
204
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
205
|
+
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
|
206
|
+
dependencies = [
|
|
207
|
+
"either",
|
|
208
|
+
"rayon-core",
|
|
209
|
+
]
|
|
210
|
+
|
|
211
|
+
[[package]]
|
|
212
|
+
name = "rayon-core"
|
|
213
|
+
version = "1.13.0"
|
|
214
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
215
|
+
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
|
216
|
+
dependencies = [
|
|
217
|
+
"crossbeam-deque",
|
|
218
|
+
"crossbeam-utils",
|
|
219
|
+
]
|
|
220
|
+
|
|
221
|
+
[[package]]
|
|
222
|
+
name = "scopeguard"
|
|
223
|
+
version = "1.2.0"
|
|
224
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
225
|
+
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
|
226
|
+
|
|
227
|
+
[[package]]
|
|
228
|
+
name = "seq-macro"
|
|
229
|
+
version = "0.3.6"
|
|
230
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
231
|
+
checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
|
|
232
|
+
|
|
233
|
+
[[package]]
|
|
234
|
+
name = "simd-adler32"
|
|
235
|
+
version = "0.3.10"
|
|
236
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
237
|
+
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
|
238
|
+
|
|
239
|
+
[[package]]
|
|
240
|
+
name = "syn"
|
|
241
|
+
version = "2.0.119"
|
|
242
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
243
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
244
|
+
dependencies = [
|
|
245
|
+
"proc-macro2",
|
|
246
|
+
"quote",
|
|
247
|
+
"unicode-ident",
|
|
248
|
+
]
|
|
249
|
+
|
|
250
|
+
[[package]]
|
|
251
|
+
name = "target-lexicon"
|
|
252
|
+
version = "0.13.5"
|
|
253
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
254
|
+
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
255
|
+
|
|
256
|
+
[[package]]
|
|
257
|
+
name = "twobitreader"
|
|
258
|
+
version = "0.2.1"
|
|
259
|
+
dependencies = [
|
|
260
|
+
"byteorder",
|
|
261
|
+
"flate2",
|
|
262
|
+
"itertools",
|
|
263
|
+
"libc",
|
|
264
|
+
"memmap2",
|
|
265
|
+
"pyo3",
|
|
266
|
+
"rayon",
|
|
267
|
+
"scopeguard",
|
|
268
|
+
"seq-macro",
|
|
269
|
+
"windows-sys",
|
|
270
|
+
]
|
|
271
|
+
|
|
272
|
+
[[package]]
|
|
273
|
+
name = "unicode-ident"
|
|
274
|
+
version = "1.0.26"
|
|
275
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
276
|
+
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
|
277
|
+
|
|
278
|
+
[[package]]
|
|
279
|
+
name = "windows-link"
|
|
280
|
+
version = "0.2.1"
|
|
281
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
282
|
+
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
|
283
|
+
|
|
284
|
+
[[package]]
|
|
285
|
+
name = "windows-sys"
|
|
286
|
+
version = "0.61.2"
|
|
287
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
288
|
+
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
|
289
|
+
dependencies = [
|
|
290
|
+
"windows-link",
|
|
291
|
+
]
|
|
292
|
+
|
|
293
|
+
[[package]]
|
|
294
|
+
name = "zlib-rs"
|
|
295
|
+
version = "0.6.8"
|
|
296
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
297
|
+
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "twobitreader"
|
|
3
|
+
version = "0.2.1"
|
|
4
|
+
edition = "2024"
|
|
5
|
+
authors = ["Andrew Delong <andrew.delong@gmail.com>"]
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
description = "Fast 2bit file reader"
|
|
8
|
+
repository = "https://github.com/andrewdelong/twobitreader-rust"
|
|
9
|
+
license = "MIT OR Apache-2.0"
|
|
10
|
+
keywords = ["2bit", "twobit", "bioinformatics", "dna", "genome"]
|
|
11
|
+
categories = ["science"]
|
|
12
|
+
exclude = ["tests/", "benches/", "doc/", "*.code-workspace", ".*", "BENCH.md"]
|
|
13
|
+
|
|
14
|
+
[features]
|
|
15
|
+
python = ["dep:pyo3", "dep:scopeguard"]
|
|
16
|
+
|
|
17
|
+
[dependencies]
|
|
18
|
+
byteorder = "1"
|
|
19
|
+
memmap2 = "0"
|
|
20
|
+
seq-macro = "0"
|
|
21
|
+
pyo3 = { version = "0", optional = true }
|
|
22
|
+
scopeguard = { version = "1", optional = true}
|
|
23
|
+
|
|
24
|
+
# Apple targets need fcntl(F_RDADVISE) to prefetch file ranges; madvise does not work there.
|
|
25
|
+
[target.'cfg(target_vendor = "apple")'.dependencies]
|
|
26
|
+
libc = "0"
|
|
27
|
+
|
|
28
|
+
# Windows targets need PrefetchVirtualMemory to prefetch file ranges.
|
|
29
|
+
[target.'cfg(windows)'.dependencies]
|
|
30
|
+
windows-sys = { version = "0", features = ["Win32_System_Memory", "Win32_System_Threading"] }
|
|
31
|
+
|
|
32
|
+
[dev-dependencies]
|
|
33
|
+
flate2 = "1"
|
|
34
|
+
itertools = ">=0.13, <1.0"
|
|
35
|
+
rayon = "1"
|
|
36
|
+
|
|
37
|
+
# Cold benchmarks use fcntl(F_NOCACHE) to write an uncached copy of the 2bit file.
|
|
38
|
+
[target.'cfg(target_vendor = "apple")'.dev-dependencies]
|
|
39
|
+
libc = "0"
|
|
40
|
+
|
|
41
|
+
[package.metadata.docs.rs]
|
|
42
|
+
rustdoc-args = ["--extend-css", "custom.css"]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Copyright (c) 2023 Andrew Delong
|
|
2
|
+
|
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
you may not use this file except in compliance with the License.
|
|
5
|
+
You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
See the License for the specific language governing permissions and
|
|
13
|
+
limitations under the License.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Copyright (c) 2023 Andrew Delong
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any
|
|
4
|
+
person obtaining a copy of this software and associated
|
|
5
|
+
documentation files (the "Software"), to deal in the
|
|
6
|
+
Software without restriction, including without
|
|
7
|
+
limitation the rights to use, copy, modify, merge,
|
|
8
|
+
publish, distribute, sublicense, and/or sell copies of
|
|
9
|
+
the Software, and to permit persons to whom the Software
|
|
10
|
+
is furnished to do so, subject to the following
|
|
11
|
+
conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice
|
|
14
|
+
shall be included in all copies or substantial portions
|
|
15
|
+
of the Software.
|
|
16
|
+
|
|
17
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
|
|
18
|
+
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
|
|
19
|
+
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
|
20
|
+
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
|
21
|
+
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
|
22
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
23
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
|
|
24
|
+
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
25
|
+
DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: twobitreader-rs
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
5
|
+
Classifier: Intended Audience :: Science/Research
|
|
6
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
7
|
+
Classifier: Operating System :: MacOS
|
|
8
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Programming Language :: Rust
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
|
+
Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Dist: pytest ; extra == 'tests'
|
|
23
|
+
Provides-Extra: tests
|
|
24
|
+
License-File: LICENSE-MIT
|
|
25
|
+
License-File: LICENSE-APACHE
|
|
26
|
+
Summary: Python bindings for the twobitreader Rust crate
|
|
27
|
+
License-Expression: MIT OR Apache-2.0
|
|
28
|
+
Requires-Python: >=3.9
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# twobitreader
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
This crate provides fast DNA sequence extraction from 2bit files, a
|
|
5
|
+
[standard format](http://genome.ucsc.edu/FAQ/FAQformat.html#format7) in bioinformatics.
|
|
6
|
+
|
|
7
|
+
The motivation for `twobitreader` is speed; see benchmarks below.
|
|
8
|
+
It is also available as a Python package `twobitreader_rs`.
|
|
9
|
+
|
|
10
|
+
The focus is raw reading from 2bit, but fast concatenation and reverse-complement methods
|
|
11
|
+
are also provided to make higher-level use cases easier.
|
|
12
|
+
|
|
13
|
+
[](https://github.com/andrewdelong/twobitreader-rust/actions/workflows/ci.yml)
|
|
14
|
+
|
|
15
|
+

|
|
16
|
+

|
|
17
|
+

|
|
18
|
+
|
|
19
|
+
## Examples
|
|
20
|
+
|
|
21
|
+
**Extracting sequences** is straightforward:
|
|
22
|
+
```rust
|
|
23
|
+
let tbr = TwobitReader::open("hg38.2bit")?; // Human genome, build 38
|
|
24
|
+
let seq = tbr.get("chr1", 10000, 10005); // -> String ("TAACC")
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
**Concatenation** works by iterating over (start, end) pairs.
|
|
28
|
+
For example, assembling a spliced transcript:
|
|
29
|
+
```rust
|
|
30
|
+
// Exon ranges for human FKHL6 gene transcript (Gencode v43)
|
|
31
|
+
let exons = [(1389575, 1391118), // Exon 1 (start, end)
|
|
32
|
+
(1394695, 1395603)]; // Exon 2 (start, end)
|
|
33
|
+
let transcript = tbr.concat("chr6", &exons); // -> String
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
**Parallelism** is easy with crates like [`rayon`](https://docs.rs/rayon/latest/rayon/).
|
|
37
|
+
For example, batch extraction of sequences:
|
|
38
|
+
```rust
|
|
39
|
+
use rayon::prelude::*;
|
|
40
|
+
let args = [("chr1", 10000, 15000),
|
|
41
|
+
("chr1", 30000, 35000), /* ... */ ];
|
|
42
|
+
let seqs = args.into_par_iter()
|
|
43
|
+
.map(|(chrom, start, end)| tbr.get(chrom, start, end))
|
|
44
|
+
.collect::<Vec<_>>(); // -> Vec<String>
|
|
45
|
+
```
|
|
46
|
+
Or, assembling a batch of spliced transcripts in parallel:
|
|
47
|
+
```rust
|
|
48
|
+
use twobitreader::reverse_complement;
|
|
49
|
+
use rayon::prelude::*;
|
|
50
|
+
|
|
51
|
+
fn stranded(seq: String, strand: char) -> String {
|
|
52
|
+
if strand == '+' { seq } else { reverse_complement(seq) }
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
let transcripts = [ // (transcript_id, chromosome, exons)
|
|
56
|
+
("ENST00000407983.7", "chr2", '+', vec![(264899, 265007), // Exon 1
|
|
57
|
+
(271865, 271939), // Exon 2
|
|
58
|
+
(272036, 272557)]), // Exon 3
|
|
59
|
+
("ENST00000319331.4", "chr3", '+', vec![(3799430, 3799919), // Exon 1
|
|
60
|
+
(3844363, 3849834)]), // Exon 2
|
|
61
|
+
/* ... */
|
|
62
|
+
];
|
|
63
|
+
// Concatenate exons and reverse-complement if negative strand.
|
|
64
|
+
// (Correct when exons are listed in genome-coordinate order.)
|
|
65
|
+
let seqs = transcripts.into_par_iter()
|
|
66
|
+
.map(|(id, chrom, strand, exons)| (id, stranded(tbr.concat(chrom, exons), strand)))
|
|
67
|
+
.collect::<HashMap<_, _>>(); // HashMap<&str, String>
|
|
68
|
+
let seq = &seqs["ENST00000407983.7"]; // -> &String to transcript sequence
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
**Cold files** are an order of magnitude slower to access than files already in memory ("hot").
|
|
72
|
+
Use prefetching to dramatically improve single-threaded speed:
|
|
73
|
+
```rust
|
|
74
|
+
let exons = [("chr1", 10000, 10200),
|
|
75
|
+
("chr1", 10500, 10700), /* ... */ ];
|
|
76
|
+
tbr.prefetch(&exons); // Ask the operating system to start paging this data from disk.
|
|
77
|
+
let seqs = tbr.get_batch(&exons); // Access the memory as it arrives.
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Benchmarks
|
|
81
|
+
|
|
82
|
+
Two tasks were benchmarked:
|
|
83
|
+
- **exons**: extract 133,388 distinct human exon sequences;
|
|
84
|
+
- **transcripts**: concatenate 319,468 exons into 29,211 human spliced transcript sequences.
|
|
85
|
+
|
|
86
|
+
Speed depends on parallelism and page cache (hot vs cold):
|
|
87
|
+
- **hot** runs represent repeated or interactive dna extraction scenarios;
|
|
88
|
+
- **cold** runs represent a first run of a genomics pipeline, bound by disk speed;
|
|
89
|
+
- **prefetch** runs are cold but with a `prefetch` call preceding extraction.
|
|
90
|
+
|
|
91
|
+
The table below shows running times in milliseconds. Experimental details are `BENCH.md`.
|
|
92
|
+
This crate provides *twobitreader* (pure rust) and *twobitreader_rs* (python wrapper).
|
|
93
|
+
|
|
94
|
+
| EXONS | 1-thread / hot | 1-thread / cold | 1-thread / prefetch | 16-thread / hot | 16-thread / cold |
|
|
95
|
+
|---|---:|---:|---:|---:|---:|
|
|
96
|
+
| **twobitreader** (rs) | 90 | 1,700 | 180 | 10 | 190 |
|
|
97
|
+
| **twobitreader_rs** (rs, py) | 100 | 1,900 | 190 | *27 | *210 |
|
|
98
|
+
| **py2bit** (c, py) | 220 | 2,800 | n/a | n/a | n/a |
|
|
99
|
+
| **GenomeKit** (cpp, py) | 340 | 2,400 | n/a | n/a | n/a |
|
|
100
|
+
| **twobit** (rs) | 390 | 3,500 | n/a | n/a | n/a |
|
|
101
|
+
| **twobitToFa** (c) | 1,000 | 4,500 | n/a | 340 | 850 |
|
|
102
|
+
| **twobitreader** (py) | 7,200 | 13,000 | n/a | 1,900 | 2,300 |
|
|
103
|
+
| **Biopython** (py) | 8,100 | 11,000 | n/a | n/a | n/a |
|
|
104
|
+
|
|
105
|
+
| TRANSCRIPTS | 1-thread / hot | 1-thread / cold | 1-thread / prefetch | 16-thread / hot | 16-thread / cold |
|
|
106
|
+
|---|---:|---:|---:|---:|---:|
|
|
107
|
+
| **twobitreader** (rs) | 140 | 1,700 | 230 | 13 | 190 |
|
|
108
|
+
| **twobitreader_rs** (rs, py) | 180 | 2,100 | 310 | *27 | *220 |
|
|
109
|
+
| **py2bit** (c, py) | 490 | 3,600 | n/a | n/a | n/a |
|
|
110
|
+
| **GenomeKit** (cpp, py) | 720 | 2,900 | n/a | n/a | n/a |
|
|
111
|
+
| **twobit** (rs) | 930 | 1,900 | n/a | n/a | n/a |
|
|
112
|
+
| **twobitToFa** (c) | 2,100 | 6,900 | n/a | 840 | 1,200 |
|
|
113
|
+
| **twobitreader** (py) | 13,000 | 21,000 | n/a | 2,800 | 3,500 |
|
|
114
|
+
| **Biopython** (py) | 19,000 | 25,000 | n/a | n/a | n/a |
|
|
115
|
+
|
|
116
|
+
Entries marked * were run in free-threaded Python.
|
|
117
|
+
|
|
118
|
+
## Dependencies
|
|
119
|
+
|
|
120
|
+
* `byteorder` for handling endian-ness
|
|
121
|
+
* `memmap2` for memory mapping the 2bit file
|
|
122
|
+
* `seq-macro` for generating 2bit decoder lookup table
|
|
123
|
+
* `libc` for prefetching file ranges on Apple targets
|
|
124
|
+
* `windows-sys` for prefetching file ranges on Windows targets
|
|
125
|
+
|
|
126
|
+
## Use of AI
|
|
127
|
+
|
|
128
|
+
Claude Code: generated the OS-specific prefetch loops; improved handling of corrupt or malicious files;
|
|
129
|
+
documented the Python bindings and mirrored their tests and benchmark; improved error checking and
|
|
130
|
+
propagation to Python more broadly; and generated the CI configurations.
|
|
131
|
+
|
|
132
|
+
License: MIT OR Apache-2.0
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Custom CSS to finesse the font sizes of inline code and block-formatted code.
|
|
3
|
+
To build docs with this CSS, first build all docs (including those of dependencies)
|
|
4
|
+
as usual, then separately build the docs for this crate with --html-in-header:
|
|
5
|
+
|
|
6
|
+
$ cargo doc
|
|
7
|
+
$ RUSTDOCFLAGS="--extend-css src/docs/custom.css" cargo doc --no-deps
|
|
8
|
+
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
body {
|
|
12
|
+
font-family: sans-serif;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/* inline code (` `) */
|
|
16
|
+
code {
|
|
17
|
+
font-size: 0.9em;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/* block formatted code (``` ```)*/
|
|
21
|
+
pre {
|
|
22
|
+
padding: 6px;
|
|
23
|
+
line-height: 1.25;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
pre code {
|
|
27
|
+
font-size: 0.9em;
|
|
28
|
+
}
|
|
File without changes
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["maturin>=1.15,<2.0"]
|
|
3
|
+
build-backend = "maturin"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "twobitreader-rs"
|
|
7
|
+
description = "Python bindings for the twobitreader Rust crate"
|
|
8
|
+
requires-python = ">=3.9"
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
|
11
|
+
"Intended Audience :: Science/Research",
|
|
12
|
+
"Operating System :: POSIX :: Linux",
|
|
13
|
+
"Operating System :: MacOS",
|
|
14
|
+
"Operating System :: Microsoft :: Windows",
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Programming Language :: Rust",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
19
|
+
"Programming Language :: Python :: 3.9",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Programming Language :: Python :: Implementation :: CPython",
|
|
26
|
+
"Programming Language :: Python :: Free Threading :: 2 - Beta",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
dynamic = ["version"]
|
|
30
|
+
license = "MIT OR Apache-2.0"
|
|
31
|
+
license-files = ["LICENSE-MIT", "LICENSE-APACHE"]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
tests = [
|
|
35
|
+
"pytest",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[tool.maturin]
|
|
39
|
+
features = ["python"]
|
|
40
|
+
module-name = "twobitreader_rs"
|