bed-reader 0.2.37__cp312-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. bed_reader/__init__.py +30 -0
  2. bed_reader/_open_bed.py +1290 -0
  3. bed_reader/_sample_data.py +99 -0
  4. bed_reader/_to_bed.py +235 -0
  5. bed_reader/bed_reader.cp312-win_amd64.pyd +0 -0
  6. bed_reader/tests/__init__.py +1 -0
  7. bed_reader/tests/benchmark/benchmark.py +129 -0
  8. bed_reader/tests/data/500x400_o640_array.memmap +0 -0
  9. bed_reader/tests/data/badfile.bed +1 -0
  10. bed_reader/tests/data/empty.bed +0 -0
  11. bed_reader/tests/data/plink_sim_10s_100v_10pmiss.bed +0 -0
  12. bed_reader/tests/data/plink_sim_10s_100v_10pmiss.bim +100 -0
  13. bed_reader/tests/data/plink_sim_10s_100v_10pmiss.fam +10 -0
  14. bed_reader/tests/data/small.bad_bim +4 -0
  15. bed_reader/tests/data/small.bed +1 -0
  16. bed_reader/tests/data/small.bim +4 -0
  17. bed_reader/tests/data/small.bim_bad_positions.bim +4 -0
  18. bed_reader/tests/data/small.deb +1 -0
  19. bed_reader/tests/data/small.fam +3 -0
  20. bed_reader/tests/data/small.fam_bad +4 -0
  21. bed_reader/tests/data/small.maf +3 -0
  22. bed_reader/tests/data/small.mib +4 -0
  23. bed_reader/tests/data/small2_array.memmap +0 -0
  24. bed_reader/tests/data/small_array.memmap +0 -0
  25. bed_reader/tests/data/small_no_bim.bed +1 -0
  26. bed_reader/tests/data/small_no_bim.fam +3 -0
  27. bed_reader/tests/data/small_no_fam.bed +1 -0
  28. bed_reader/tests/data/small_no_fam.bim +4 -0
  29. bed_reader/tests/data/small_too_short.bed +1 -0
  30. bed_reader/tests/data/small_too_short.bim +4 -0
  31. bed_reader/tests/data/small_too_short.fam +3 -0
  32. bed_reader/tests/data/smallmode0.bed +0 -0
  33. bed_reader/tests/data/smallmode0.bim +3 -0
  34. bed_reader/tests/data/smallmode0.fam +4 -0
  35. bed_reader/tests/data/smallmodebad.bed +1 -0
  36. bed_reader/tests/data/some_missing.bed +0 -0
  37. bed_reader/tests/data/some_missing.bim +100 -0
  38. bed_reader/tests/data/some_missing.fam +100 -0
  39. bed_reader/tests/data/some_missing.properties.npz +0 -0
  40. bed_reader/tests/data/some_missing.val.npy +0 -0
  41. bed_reader/tests/data/toydata.5chrom.bed +0 -0
  42. bed_reader/tests/data/toydata.5chrom.bim +10000 -0
  43. bed_reader/tests/data/toydata.5chrom.fam +500 -0
  44. bed_reader/tests/registry.txt +36 -0
  45. bed_reader/tests/test_file_dot.py +368 -0
  46. bed_reader/tests/test_open_bed.py +806 -0
  47. bed_reader-0.2.37.dist-info/METADATA +112 -0
  48. bed_reader-0.2.37.dist-info/RECORD +51 -0
  49. bed_reader-0.2.37.dist-info/WHEEL +4 -0
  50. bed_reader-0.2.37.dist-info/license_files/AUTHORS.txt +2 -0
  51. bed_reader-0.2.37.dist-info/license_files/LICENSE.md +202 -0
@@ -0,0 +1,368 @@
1
+ import logging
2
+ from datetime import datetime
3
+ from pathlib import Path
4
+
5
+ import numpy as np
6
+ import pytest
7
+
8
+ from bed_reader import (
9
+ file_aat_piece_f64_orderf,
10
+ file_ata_piece_f64_orderf,
11
+ file_b_less_aatbx,
12
+ file_dot_piece,
13
+ )
14
+ from bed_reader._open_bed import get_num_threads, open_bed # noqa
15
+
16
+
17
+ def file_ata(filename, offset, iid_count, sid_count, sid_step):
18
+ ata = np.full((sid_count, sid_count), np.nan)
19
+ for sid_index in range(0, sid_count, sid_step):
20
+ sid_range_len = min(sid_step, sid_count - sid_index)
21
+ ata_piece = np.full((sid_count - sid_index, sid_range_len), np.nan)
22
+ if sid_index % 2 == 0: # test new and old method
23
+ file_ata_piece_f64_orderf(
24
+ str(filename),
25
+ offset,
26
+ iid_count,
27
+ sid_count,
28
+ sid_index,
29
+ ata_piece,
30
+ num_threads=get_num_threads(None),
31
+ log_frequency=sid_range_len,
32
+ )
33
+ else:
34
+ file_dot_piece(
35
+ str(filename),
36
+ offset,
37
+ iid_count,
38
+ sid_index,
39
+ ata_piece,
40
+ num_threads=get_num_threads(None),
41
+ log_frequency=sid_range_len,
42
+ )
43
+ ata[sid_index:, sid_index : sid_index + sid_range_len] = ata_piece
44
+ for sid_index in range(sid_count):
45
+ ata[sid_index, sid_index + 1 :] = ata[sid_index + 1 :, sid_index]
46
+ return ata
47
+
48
+
49
+ def file_aat(filename, offset, iid_count, sid_count, iid_step):
50
+ aat = np.full((iid_count, iid_count), np.nan)
51
+ for iid_index in range(0, iid_count, iid_step):
52
+ iid_range_len = min(iid_step, iid_count - iid_index)
53
+ aat_piece = np.full((iid_count - iid_index, iid_range_len), np.nan)
54
+ file_aat_piece_f64_orderf(
55
+ str(filename),
56
+ offset,
57
+ iid_count,
58
+ sid_count,
59
+ iid_index,
60
+ aat_piece,
61
+ num_threads=get_num_threads(None),
62
+ log_frequency=iid_range_len,
63
+ )
64
+ aat[iid_index:, iid_index : iid_index + iid_range_len] = aat_piece
65
+ for iid_index in range(iid_count):
66
+ aat[iid_index, iid_index + 1 :] = aat[iid_index + 1 :, iid_index]
67
+
68
+ return aat
69
+
70
+
71
+ def write_read_test_file_ata(iid_count, sid_count, sid_step, tmp_path):
72
+ offset = 640
73
+ file_path = tmp_path / f"{iid_count}x{sid_count}_o{offset}_array.memmap"
74
+
75
+ mm = np.memmap(
76
+ file_path,
77
+ dtype="float64",
78
+ mode="w+",
79
+ offset=offset,
80
+ shape=(iid_count, sid_count),
81
+ order="F",
82
+ )
83
+ mm[:] = np.linspace(0, 1, mm.size).reshape(mm.shape)
84
+ mm.flush()
85
+
86
+ out_val = file_ata(file_path, offset, iid_count, sid_count, sid_step)
87
+ expected = mm.T.dot(mm)
88
+ assert np.allclose(expected, out_val, equal_nan=True)
89
+
90
+
91
+ def write_read_test_file_aat(
92
+ iid_count, sid_count, iid_step, tmp_path, skip_if_there=False
93
+ ):
94
+ offset = 640
95
+ file_path = tmp_path / f"{iid_count}x{sid_count}_o{offset}_array.memmap"
96
+ if skip_if_there and file_path.exists():
97
+ logging.info(f"'{file_path}' exists")
98
+ mm = np.memmap(
99
+ file_path,
100
+ dtype="float64",
101
+ mode="r",
102
+ offset=offset,
103
+ shape=(iid_count, sid_count),
104
+ order="F",
105
+ )
106
+ else:
107
+ logging.info(f"Creating '{file_path}'")
108
+ mm = np.memmap(
109
+ file_path,
110
+ dtype="float64",
111
+ mode="w+",
112
+ offset=offset,
113
+ shape=(iid_count, sid_count),
114
+ order="F",
115
+ )
116
+ mm[:] = np.linspace(0, 1, mm.size).reshape(mm.shape)
117
+ mm.flush()
118
+ logging.info(f"Finished creating '{file_path}'")
119
+
120
+ start_time = datetime.now()
121
+ out_val = file_aat(file_path, offset, iid_count, sid_count, iid_step)
122
+ delta = datetime.now() - start_time
123
+ expected = mm.dot(mm.T)
124
+ assert np.allclose(expected, out_val, equal_nan=True)
125
+ return delta
126
+
127
+
128
+ def test_file_ata_medium(tmp_path):
129
+ write_read_test_file_ata(100, 1000, 33, tmp_path)
130
+
131
+
132
+ def test_file_aat_medium(tmp_path):
133
+ write_read_test_file_aat(100, 1000, 34, tmp_path)
134
+
135
+
136
+ # # Too slow
137
+ # def test_file_ata_giant(tmp_path):
138
+ # write_read_test_file_ata(100_000, 10_000, 1000, tmp_path)
139
+
140
+ # # Too slow
141
+ # def test_file_aat_giant(tmp_path):
142
+ # write_read_test_file_ata(1_000, 10_000, 100, tmp_path)
143
+
144
+
145
+ def test_file_ata_small(shared_datadir):
146
+ filename = shared_datadir / "small_array.memmap"
147
+
148
+ out_val = file_ata(filename, 0, 2, 3, 2)
149
+ print(out_val)
150
+
151
+ expected = np.array([[17.0, 22.0, 27.0], [22.0, 29.0, 36.0], [27.0, 36.0, 45.0]])
152
+ print(expected)
153
+ assert np.allclose(expected, out_val, equal_nan=True)
154
+
155
+
156
+ def test_file_aat_small(shared_datadir):
157
+ filename = shared_datadir / "small_array.memmap"
158
+
159
+ out_val = file_aat(filename, 0, iid_count=2, sid_count=3, iid_step=1)
160
+ print(out_val)
161
+
162
+ expected = np.array([[14.0, 32.0], [32.0, 77.0]])
163
+ print(expected)
164
+ assert np.allclose(expected, out_val, equal_nan=True)
165
+
166
+
167
+ def mmultfile_b_less_aatb(a_snp_mem_map, b, log_frequency=0, force_python_only=False):
168
+ # Without memory efficiency
169
+ # a=a_snp_mem_map.val
170
+ # aTb = np.dot(a.T,b)
171
+ # aaTb = b-np.dot(a,aTb)
172
+ # return aTb, aaTb
173
+
174
+ if force_python_only:
175
+ aTb = np.zeros(
176
+ (a_snp_mem_map.shape[1], b.shape[1])
177
+ ) # b can be destroyed. Is everything is in best order, i.e. F vs C
178
+ aaTb = b.copy()
179
+ b_mem = np.array(b, order="F")
180
+ with open(a_snp_mem_map.filename, "rb") as U_fp:
181
+ U_fp.seek(a_snp_mem_map.offset)
182
+ for i in range(a_snp_mem_map.shape[1]):
183
+ a_mem = np.fromfile(
184
+ U_fp, dtype=np.float64, count=a_snp_mem_map.shape[0]
185
+ )
186
+ if log_frequency > 0 and i % log_frequency == 0:
187
+ logging.info("{0}/{1}".format(i, a_snp_mem_map.shape[1]))
188
+ aTb[i, :] = np.dot(a_mem, b_mem)
189
+ aaTb -= np.dot(a_mem.reshape(-1, 1), aTb[i : i + 1, :])
190
+ else:
191
+ b1 = np.array(b, order="F")
192
+ aTb = np.zeros((a_snp_mem_map.shape[1], b.shape[1]))
193
+ aaTb = np.array(b1, order="F")
194
+
195
+ file_b_less_aatbx(
196
+ str(a_snp_mem_map.filename),
197
+ a_snp_mem_map.offset,
198
+ a_snp_mem_map.shape[0], # row count
199
+ b1, # B copy 1 in "F" order
200
+ aaTb, # B copy 2 in "F" order
201
+ aTb, # result
202
+ num_threads=get_num_threads(None),
203
+ log_frequency=log_frequency,
204
+ )
205
+
206
+ return aTb, aaTb
207
+
208
+
209
+ def write_read_test_file_b_less_aatbx(
210
+ iid_count, a_sid_count, b_sid_count, log_frequency, tmp_path, do_both=True
211
+ ):
212
+ offset = 640
213
+ file_path = tmp_path / f"{iid_count}x{a_sid_count}_o{offset}_array.memmap"
214
+ mm = np.memmap(
215
+ file_path,
216
+ dtype="float64",
217
+ mode="w+",
218
+ offset=offset,
219
+ shape=(iid_count, a_sid_count),
220
+ order="F",
221
+ )
222
+ mm[:] = np.linspace(0, 1, mm.size).reshape(mm.shape)
223
+ mm.flush()
224
+
225
+ b = np.array(
226
+ np.linspace(0, 1, iid_count * b_sid_count).reshape(
227
+ (iid_count, b_sid_count), order="F"
228
+ )
229
+ )
230
+ b_again = b.copy()
231
+
232
+ logging.info("Calling Rust")
233
+ aTb, aaTb = mmultfile_b_less_aatb(
234
+ mm, b_again, log_frequency, force_python_only=False
235
+ )
236
+
237
+ if do_both:
238
+ logging.info("Calling Python")
239
+ aTb_python, aaTb_python = mmultfile_b_less_aatb(
240
+ mm, b, log_frequency, force_python_only=True
241
+ )
242
+
243
+ if (
244
+ not np.abs(aTb_python - aTb).max() < 1e-8
245
+ or not np.abs(aaTb_python - aaTb).max() < 1e-8
246
+ ):
247
+ raise AssertionError(
248
+ "Expect Python and Rust to get the same mmultfile_b_less_aatb answer"
249
+ )
250
+
251
+
252
+ def test_file_b_less_aatbx_medium(tmp_path):
253
+ write_read_test_file_b_less_aatbx(500, 400, 100, 10, tmp_path, do_both=True)
254
+
255
+
256
+ def test_file_b_less_aatbx_medium2(tmp_path):
257
+ write_read_test_file_b_less_aatbx(5_000, 400, 100, 100, tmp_path, do_both=True)
258
+
259
+
260
+ # Slow and doesn't check answer
261
+ # def test_file_b_less_aatbx_2(tmp_path):
262
+ # write_read_test_file_b_less_aatbx(50_000, 4000, 1000, 100, tmp_path, do_both=False)
263
+
264
+
265
+ if __name__ == "__main__":
266
+ logging.basicConfig(level=logging.INFO)
267
+ shared_datadir = Path(r"D:\OneDrive\programs\bed-reader\bed_reader\tests\data")
268
+ tmp_path = Path(r"m:/deldir/tests")
269
+
270
+ if False:
271
+ small2_array = np.array(
272
+ [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0]],
273
+ order="F",
274
+ )
275
+ mm = np.memmap(
276
+ tmp_path / "small2_array.memmap",
277
+ dtype="float64",
278
+ mode="w+",
279
+ shape=(3, 4),
280
+ order="F",
281
+ )
282
+ mm[:] = small2_array[:]
283
+ print(mm.offset)
284
+ mm.flush()
285
+
286
+ if False:
287
+ small_array = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], order="F")
288
+ mm = np.memmap(
289
+ tmp_path / "small_array.memmap",
290
+ dtype="float64",
291
+ mode="w+",
292
+ shape=(2, 3),
293
+ order="F",
294
+ )
295
+ mm[:] = small_array[:]
296
+ print(mm.offset)
297
+ mm.flush()
298
+
299
+ if False:
300
+ mm = np.memmap(
301
+ tmp_path / "100x1000_o640_array.memmap",
302
+ dtype="float64",
303
+ mode="w+",
304
+ offset=640,
305
+ shape=(100, 1000),
306
+ order="F",
307
+ )
308
+ total = mm.shape[0] * mm.shape[1]
309
+ lin = np.linspace(0, 1, total).reshape(mm.shape)
310
+
311
+ mm[:] = lin[:]
312
+ print(mm.offset)
313
+ mm.flush()
314
+
315
+ if False:
316
+ mm = np.memmap(
317
+ tmp_path / "1000x10000_o640_array.memmap",
318
+ dtype="float64",
319
+ mode="w+",
320
+ offset=640,
321
+ shape=(1000, 10_000),
322
+ order="F",
323
+ )
324
+ total = mm.shape[0] * mm.shape[1]
325
+ lin = np.linspace(0, 1, total).reshape(mm.shape)
326
+
327
+ mm[:] = lin[:]
328
+ print(mm.offset)
329
+ mm.flush()
330
+
331
+ if False:
332
+ mm = np.memmap(
333
+ tmp_path / "10_000x100_000_o6400_array.memmap",
334
+ dtype="float64",
335
+ mode="w+",
336
+ offset=640,
337
+ shape=(10_000, 100_000),
338
+ order="F",
339
+ )
340
+ total = mm.shape[0] * mm.shape[1]
341
+ lin = np.linspace(0, 1, total).reshape(mm.shape)
342
+
343
+ mm[:] = lin[:]
344
+ print(mm.offset)
345
+ mm.flush()
346
+
347
+ if False:
348
+ mm = np.memmap(
349
+ tmp_path / "100_000x10_000_o6400_array.memmap",
350
+ dtype="float64",
351
+ mode="w+",
352
+ offset=640,
353
+ shape=(100_000, 10_000),
354
+ order="F",
355
+ )
356
+ total = mm.shape[0] * mm.shape[1]
357
+ lin = np.linspace(0, 1, total).reshape(mm.shape)
358
+
359
+ mm[:] = lin[:]
360
+ print(mm.offset)
361
+ mm.flush()
362
+
363
+ # test_file_b_less_aatbx_2(tmp_path)
364
+ # test_zero_files(tmp_path)
365
+ # test_index(shared_datadir)
366
+ # test_c_reader_bed(shared_datadir)
367
+ # test_read1(shared_datadir)
368
+ pytest.main([__file__])