bed-reader 0.2.37__cp312-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bed_reader/__init__.py +30 -0
- bed_reader/_open_bed.py +1290 -0
- bed_reader/_sample_data.py +99 -0
- bed_reader/_to_bed.py +235 -0
- bed_reader/bed_reader.cp312-win_amd64.pyd +0 -0
- bed_reader/tests/__init__.py +1 -0
- bed_reader/tests/benchmark/benchmark.py +129 -0
- bed_reader/tests/data/500x400_o640_array.memmap +0 -0
- bed_reader/tests/data/badfile.bed +1 -0
- bed_reader/tests/data/empty.bed +0 -0
- bed_reader/tests/data/plink_sim_10s_100v_10pmiss.bed +0 -0
- bed_reader/tests/data/plink_sim_10s_100v_10pmiss.bim +100 -0
- bed_reader/tests/data/plink_sim_10s_100v_10pmiss.fam +10 -0
- bed_reader/tests/data/small.bad_bim +4 -0
- bed_reader/tests/data/small.bed +1 -0
- bed_reader/tests/data/small.bim +4 -0
- bed_reader/tests/data/small.bim_bad_positions.bim +4 -0
- bed_reader/tests/data/small.deb +1 -0
- bed_reader/tests/data/small.fam +3 -0
- bed_reader/tests/data/small.fam_bad +4 -0
- bed_reader/tests/data/small.maf +3 -0
- bed_reader/tests/data/small.mib +4 -0
- bed_reader/tests/data/small2_array.memmap +0 -0
- bed_reader/tests/data/small_array.memmap +0 -0
- bed_reader/tests/data/small_no_bim.bed +1 -0
- bed_reader/tests/data/small_no_bim.fam +3 -0
- bed_reader/tests/data/small_no_fam.bed +1 -0
- bed_reader/tests/data/small_no_fam.bim +4 -0
- bed_reader/tests/data/small_too_short.bed +1 -0
- bed_reader/tests/data/small_too_short.bim +4 -0
- bed_reader/tests/data/small_too_short.fam +3 -0
- bed_reader/tests/data/smallmode0.bed +0 -0
- bed_reader/tests/data/smallmode0.bim +3 -0
- bed_reader/tests/data/smallmode0.fam +4 -0
- bed_reader/tests/data/smallmodebad.bed +1 -0
- bed_reader/tests/data/some_missing.bed +0 -0
- bed_reader/tests/data/some_missing.bim +100 -0
- bed_reader/tests/data/some_missing.fam +100 -0
- bed_reader/tests/data/some_missing.properties.npz +0 -0
- bed_reader/tests/data/some_missing.val.npy +0 -0
- bed_reader/tests/data/toydata.5chrom.bed +0 -0
- bed_reader/tests/data/toydata.5chrom.bim +10000 -0
- bed_reader/tests/data/toydata.5chrom.fam +500 -0
- bed_reader/tests/registry.txt +36 -0
- bed_reader/tests/test_file_dot.py +368 -0
- bed_reader/tests/test_open_bed.py +806 -0
- bed_reader-0.2.37.dist-info/METADATA +112 -0
- bed_reader-0.2.37.dist-info/RECORD +51 -0
- bed_reader-0.2.37.dist-info/WHEEL +4 -0
- bed_reader-0.2.37.dist-info/license_files/AUTHORS.txt +2 -0
- bed_reader-0.2.37.dist-info/license_files/LICENSE.md +202 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import tempfile
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Union
|
|
4
|
+
|
|
5
|
+
import pooch
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
Load sample data.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
POOCH = pooch.create(
|
|
12
|
+
# Use the default cache folder for the OS
|
|
13
|
+
path=pooch.os_cache("bed_reader"),
|
|
14
|
+
# The remote data is on Github
|
|
15
|
+
base_url="https://raw.githubusercontent.com/"
|
|
16
|
+
+ "fastlmm/bed-reader/rustybed/bed_reader/tests/data/",
|
|
17
|
+
# If this is a development version, get the data from the master branch
|
|
18
|
+
version_dev="master",
|
|
19
|
+
# The registry specifies the files that can be fetched
|
|
20
|
+
env="BED_READER_DATA_DIR",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
# Get registry file from package_data
|
|
24
|
+
registry_file = Path(__file__).parent / "tests/registry.txt"
|
|
25
|
+
# Load this registry file
|
|
26
|
+
POOCH.load_registry(registry_file)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def sample_file(filepath: Union[str, Path]) -> str:
|
|
30
|
+
"""
|
|
31
|
+
Retrieve a sample .bed file. (Also retrieves associated .fam and .bim files).
|
|
32
|
+
|
|
33
|
+
Parameters
|
|
34
|
+
----------
|
|
35
|
+
filepath
|
|
36
|
+
Name of the sample .bed file.
|
|
37
|
+
|
|
38
|
+
Returns
|
|
39
|
+
-------
|
|
40
|
+
str
|
|
41
|
+
Local name of sample .bed file.
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
By default this function puts files under the user's cache directory.
|
|
45
|
+
Override this by setting
|
|
46
|
+
the `BED_READER_DATA_DIR` environment variable.
|
|
47
|
+
|
|
48
|
+
Example
|
|
49
|
+
--------
|
|
50
|
+
|
|
51
|
+
.. doctest::
|
|
52
|
+
|
|
53
|
+
>>> from bed_reader import sample_file
|
|
54
|
+
>>>
|
|
55
|
+
>>> file_name = sample_file("small.bed")
|
|
56
|
+
>>> print(f"The local file name is '{file_name}'")
|
|
57
|
+
The local file name is '...small.bed'
|
|
58
|
+
"""
|
|
59
|
+
filepath = Path(filepath)
|
|
60
|
+
file_string = str(filepath)
|
|
61
|
+
if file_string.lower().endswith(".bed"):
|
|
62
|
+
POOCH.fetch(file_string[:-4] + ".fam")
|
|
63
|
+
POOCH.fetch(file_string[:-4] + ".bim")
|
|
64
|
+
return POOCH.fetch(file_string)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def tmp_path() -> Path:
|
|
68
|
+
"""
|
|
69
|
+
Return a :class:`pathlib.Path` to a temporary directory.
|
|
70
|
+
|
|
71
|
+
Returns
|
|
72
|
+
-------
|
|
73
|
+
pathlib.Path
|
|
74
|
+
a temporary directory
|
|
75
|
+
|
|
76
|
+
Example
|
|
77
|
+
--------
|
|
78
|
+
|
|
79
|
+
.. doctest::
|
|
80
|
+
|
|
81
|
+
>>> from bed_reader import to_bed, tmp_path
|
|
82
|
+
>>>
|
|
83
|
+
>>> output_file = tmp_path() / "small3.bed"
|
|
84
|
+
>>> val = [[1, 0, -127, 0], [2, 0, -127, 2], [0, 1, 2, 0]]
|
|
85
|
+
>>> to_bed(output_file, val)
|
|
86
|
+
|
|
87
|
+
"""
|
|
88
|
+
temp_dir = tempfile.gettempdir()
|
|
89
|
+
path = Path(temp_dir) / "bed_reader_tmp_path"
|
|
90
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
return path
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# if __name__ == "__main__":
|
|
95
|
+
# logging.basicConfig(level=logging.INFO)
|
|
96
|
+
|
|
97
|
+
# import pytest
|
|
98
|
+
|
|
99
|
+
# pytest.main(["--doctest-modules", __file__])
|
bed_reader/_to_bed.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, List, Mapping, Union
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from bed_reader import get_num_threads, open_bed
|
|
9
|
+
|
|
10
|
+
from .bed_reader import write_f32, write_f64, write_i8 # type: ignore
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def to_bed(
|
|
14
|
+
filepath: Union[str, Path],
|
|
15
|
+
val: np.ndarray,
|
|
16
|
+
properties: Mapping[str, List[Any]] = {},
|
|
17
|
+
count_A1: bool = True,
|
|
18
|
+
fam_filepath: Union[str, Path] = None,
|
|
19
|
+
bim_filepath: Union[str, Path] = None,
|
|
20
|
+
force_python_only: bool = False,
|
|
21
|
+
num_threads=None,
|
|
22
|
+
):
|
|
23
|
+
"""
|
|
24
|
+
Write values to a file in PLINK .bed format.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
filepath:
|
|
29
|
+
.bed file to write to.
|
|
30
|
+
val: array-like:
|
|
31
|
+
A two-dimension array (or array-like object) of values. The values should
|
|
32
|
+
be (or be convertible to) all floats or all integers.
|
|
33
|
+
The values should be 0, 1, 2, or missing.
|
|
34
|
+
If floats, missing is ``np.nan``. If integers, missing is -127.
|
|
35
|
+
properties: dict, optional
|
|
36
|
+
A dictionary of property names and values to write to the .fam and .bim files.
|
|
37
|
+
Any properties not mentioned will be filled in with default values.
|
|
38
|
+
|
|
39
|
+
The possible property names are:
|
|
40
|
+
|
|
41
|
+
"fid" (family id), "iid" (individual or sample id), "father" (father id),
|
|
42
|
+
"mother" (mother id), "sex", "pheno" (phenotype), "chromosome", "sid"
|
|
43
|
+
(SNP or variant id), "cm_position" (centimorgan position), "bp_position"
|
|
44
|
+
(base-pair position), "allele_1", "allele_2".
|
|
45
|
+
|
|
46
|
+
The values are lists or arrays. See example, below.
|
|
47
|
+
count_A1: bool, optional
|
|
48
|
+
True (default) to count the number of A1 alleles (the PLINK standard).
|
|
49
|
+
False to count the number of A2 alleles.
|
|
50
|
+
fam_filepath: pathlib.Path or str, optional
|
|
51
|
+
Path to the file containing information about each individual (sample).
|
|
52
|
+
Defaults to replacing the .bed file’s suffix with .fam.
|
|
53
|
+
bim_filepath: pathlib.Path or str, optional
|
|
54
|
+
Path to the file containing information about each SNP (variant).
|
|
55
|
+
Defaults to replacing the .bed file’s suffix with .bim.
|
|
56
|
+
force_python_only
|
|
57
|
+
If False (default), uses the faster Rust code; otherwise it uses the slower
|
|
58
|
+
pure Python code.
|
|
59
|
+
|
|
60
|
+
num_threads: None or int, optional
|
|
61
|
+
The number of threads with which to write data.
|
|
62
|
+
Defaults to all available processors.
|
|
63
|
+
Can also be set with these environment variables (listed in priority order):
|
|
64
|
+
'PST_NUM_THREADS', 'NUM_THREADS', 'MKL_NUM_THREADS'.
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
Examples
|
|
68
|
+
--------
|
|
69
|
+
|
|
70
|
+
In this example, all properties are given.
|
|
71
|
+
|
|
72
|
+
.. doctest::
|
|
73
|
+
|
|
74
|
+
>>> import numpy as np
|
|
75
|
+
>>> from bed_reader import to_bed, tmp_path
|
|
76
|
+
>>>
|
|
77
|
+
>>> output_file = tmp_path() / "small.bed"
|
|
78
|
+
>>> val = [[1.0, 0.0, np.nan, 0.0],
|
|
79
|
+
... [2.0, 0.0, np.nan, 2.0],
|
|
80
|
+
... [0.0, 1.0, 2.0, 0.0]]
|
|
81
|
+
>>> properties = {
|
|
82
|
+
... "fid": ["fid1", "fid1", "fid2"],
|
|
83
|
+
... "iid": ["iid1", "iid2", "iid3"],
|
|
84
|
+
... "father": ["iid23", "iid23", "iid22"],
|
|
85
|
+
... "mother": ["iid34", "iid34", "iid33"],
|
|
86
|
+
... "sex": [1, 2, 0],
|
|
87
|
+
... "pheno": ["red", "red", "blue"],
|
|
88
|
+
... "chromosome": ["1", "1", "5", "Y"],
|
|
89
|
+
... "sid": ["sid1", "sid2", "sid3", "sid4"],
|
|
90
|
+
... "cm_position": [100.4, 2000.5, 4000.7, 7000.9],
|
|
91
|
+
... "bp_position": [1, 100, 1000, 1004],
|
|
92
|
+
... "allele_1": ["A", "T", "A", "T"],
|
|
93
|
+
... "allele_2": ["A", "C", "C", "G"],
|
|
94
|
+
... }
|
|
95
|
+
>>> to_bed(output_file, val, properties=properties)
|
|
96
|
+
|
|
97
|
+
Here, no properties are given, so default values are assigned.
|
|
98
|
+
If we then read the new file and list the chromosome property,
|
|
99
|
+
it is an array of '0's, the default chromosome value.
|
|
100
|
+
|
|
101
|
+
.. doctest::
|
|
102
|
+
|
|
103
|
+
>>> output_file2 = tmp_path() / "small2.bed"
|
|
104
|
+
>>> val = [[1, 0, -127, 0], [2, 0, -127, 2], [0, 1, 2, 0]]
|
|
105
|
+
>>> to_bed(output_file2, val)
|
|
106
|
+
>>>
|
|
107
|
+
>>> from bed_reader import open_bed
|
|
108
|
+
>>> with open_bed(output_file2) as bed2:
|
|
109
|
+
... print(bed2.chromosome)
|
|
110
|
+
['0' '0' '0' '0']
|
|
111
|
+
|
|
112
|
+
"""
|
|
113
|
+
filepath = Path(filepath)
|
|
114
|
+
|
|
115
|
+
val = _fix_up_val(val)
|
|
116
|
+
iid_count = val.shape[0]
|
|
117
|
+
sid_count = val.shape[1]
|
|
118
|
+
|
|
119
|
+
properties, _ = open_bed._fix_up_properties(
|
|
120
|
+
properties, iid_count=iid_count, sid_count=sid_count, use_fill_sequence=True
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
open_bed._write_fam_or_bim(filepath, properties, "fam", fam_filepath)
|
|
124
|
+
open_bed._write_fam_or_bim(filepath, properties, "bim", bim_filepath)
|
|
125
|
+
|
|
126
|
+
if not force_python_only:
|
|
127
|
+
if not val.flags["C_CONTIGUOUS"] and not val.flags["F_CONTIGUOUS"]:
|
|
128
|
+
raise ValueError("val must be contiguous.")
|
|
129
|
+
|
|
130
|
+
num_threads = get_num_threads(num_threads)
|
|
131
|
+
|
|
132
|
+
iid_count, sid_count = val.shape
|
|
133
|
+
try:
|
|
134
|
+
if val.dtype == np.float64:
|
|
135
|
+
write_f64(
|
|
136
|
+
str(filepath),
|
|
137
|
+
is_a1_counted=count_A1,
|
|
138
|
+
val=val,
|
|
139
|
+
num_threads=num_threads,
|
|
140
|
+
)
|
|
141
|
+
elif val.dtype == np.float32:
|
|
142
|
+
write_f32(
|
|
143
|
+
str(filepath),
|
|
144
|
+
is_a1_counted=count_A1,
|
|
145
|
+
val=val,
|
|
146
|
+
num_threads=num_threads,
|
|
147
|
+
)
|
|
148
|
+
elif val.dtype == np.int8:
|
|
149
|
+
write_i8(
|
|
150
|
+
str(filepath),
|
|
151
|
+
is_a1_counted=count_A1,
|
|
152
|
+
val=val,
|
|
153
|
+
num_threads=num_threads,
|
|
154
|
+
)
|
|
155
|
+
else:
|
|
156
|
+
raise ValueError(
|
|
157
|
+
f"dtype '{val.dtype}' not known, only "
|
|
158
|
+
+ "'int8', 'float32', and 'float64' are allowed."
|
|
159
|
+
)
|
|
160
|
+
except SystemError as system_error:
|
|
161
|
+
try:
|
|
162
|
+
os.unlink(filepath)
|
|
163
|
+
except Exception:
|
|
164
|
+
pass
|
|
165
|
+
raise system_error.__cause__
|
|
166
|
+
else:
|
|
167
|
+
if not count_A1:
|
|
168
|
+
zero_code = 0b00
|
|
169
|
+
two_code = 0b11
|
|
170
|
+
else:
|
|
171
|
+
zero_code = 0b11
|
|
172
|
+
two_code = 0b00
|
|
173
|
+
|
|
174
|
+
with open(filepath, "wb") as bed_filepointer:
|
|
175
|
+
# see http://zzz.bwh.harvard.edu/plink/binary.shtml
|
|
176
|
+
bed_filepointer.write(bytes(bytearray([0b01101100]))) # magic numbers
|
|
177
|
+
bed_filepointer.write(bytes(bytearray([0b00011011]))) # magic numbers
|
|
178
|
+
bed_filepointer.write(bytes(bytearray([0b00000001]))) # snp major
|
|
179
|
+
|
|
180
|
+
for sid_index in range(sid_count):
|
|
181
|
+
if sid_index % 1 == 0:
|
|
182
|
+
logging.info(
|
|
183
|
+
"Writing snp # {0} to file '{1}'".format(sid_index, filepath)
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
col = val[:, sid_index]
|
|
187
|
+
for iid_by_four in range(0, iid_count, 4):
|
|
188
|
+
vals_for_this_byte = col[iid_by_four : iid_by_four + 4]
|
|
189
|
+
byte = 0b00000000
|
|
190
|
+
for val_index in range(len(vals_for_this_byte)):
|
|
191
|
+
val_for_byte = vals_for_this_byte[val_index]
|
|
192
|
+
if val_for_byte == 0:
|
|
193
|
+
code = zero_code
|
|
194
|
+
elif val_for_byte == 1:
|
|
195
|
+
code = 0b10 # backwards on purpose
|
|
196
|
+
elif val_for_byte == 2:
|
|
197
|
+
code = two_code
|
|
198
|
+
elif (
|
|
199
|
+
val.dtype == np.int8 and val_for_byte == -127
|
|
200
|
+
) or np.isnan(val_for_byte):
|
|
201
|
+
code = 0b01 # backwards on purpose
|
|
202
|
+
else:
|
|
203
|
+
raise ValueError(
|
|
204
|
+
"Attempt to write illegal value to .bed file. "
|
|
205
|
+
+ "Only 0,1,2,missing allowed."
|
|
206
|
+
)
|
|
207
|
+
byte |= code << (val_index * 2)
|
|
208
|
+
bed_filepointer.write(bytes(bytearray([byte])))
|
|
209
|
+
logging.info(f"Done writing {filepath}")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _fix_up_val(input):
|
|
213
|
+
if not isinstance(input, np.ndarray):
|
|
214
|
+
return _fix_up_val(np.array(input))
|
|
215
|
+
|
|
216
|
+
if np.issubdtype(input.dtype, np.integer) and input.dtype != np.int8:
|
|
217
|
+
return _fix_up_val(np.array(input, dtype=np.int8))
|
|
218
|
+
elif np.issubdtype(input.dtype, np.floating) and input.dtype not in (
|
|
219
|
+
np.float32,
|
|
220
|
+
np.float64,
|
|
221
|
+
):
|
|
222
|
+
return _fix_up_val(np.array(input, dtype=np.float32))
|
|
223
|
+
|
|
224
|
+
if len(input.shape) != 2:
|
|
225
|
+
raise ValueError("val should be two dimensional")
|
|
226
|
+
|
|
227
|
+
return input
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
# if __name__ == "__main__":
|
|
231
|
+
# logging.basicConfig(level=logging.INFO)
|
|
232
|
+
|
|
233
|
+
# import pytest
|
|
234
|
+
|
|
235
|
+
# pytest.main(["--doctest-modules", __file__])
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from bed_reader._open_bed import open_bed # noqa
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from bed_reader import to_bed
|
|
8
|
+
|
|
9
|
+
if True:
|
|
10
|
+
ssd_path = Path("m:/deldir/bench")
|
|
11
|
+
hdd_path = Path("e:/deldir/bench")
|
|
12
|
+
else:
|
|
13
|
+
ssd_path = Path("/mnt/m/deldir/bench")
|
|
14
|
+
hdd_path = Path("/mnt/e/deldir/bench")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_writes(iid_count, sid_count, num_threads, drive, include_error):
|
|
18
|
+
if drive == "ssd":
|
|
19
|
+
path = ssd_path
|
|
20
|
+
elif drive == "hdd":
|
|
21
|
+
path = hdd_path
|
|
22
|
+
else:
|
|
23
|
+
raise ValueError(f"drive must be 'ssd' or 'hdd', not '{drive}'")
|
|
24
|
+
|
|
25
|
+
output_file = path / f"{iid_count}x{sid_count}.bed"
|
|
26
|
+
|
|
27
|
+
val = np.full((iid_count, sid_count), 1, dtype=np.int8)
|
|
28
|
+
if include_error:
|
|
29
|
+
val[iid_count // 2, sid_count // 2] = 22
|
|
30
|
+
|
|
31
|
+
val_size = float(iid_count) * sid_count
|
|
32
|
+
if val_size > 9_200_000_000:
|
|
33
|
+
raise ValueError(f"val_size {val_size} is too large")
|
|
34
|
+
|
|
35
|
+
result_list = []
|
|
36
|
+
start = time.time()
|
|
37
|
+
to_bed(output_file, val, num_threads=num_threads)
|
|
38
|
+
delta = time.time() - start
|
|
39
|
+
result = [
|
|
40
|
+
iid_count,
|
|
41
|
+
sid_count,
|
|
42
|
+
num_threads,
|
|
43
|
+
drive,
|
|
44
|
+
include_error,
|
|
45
|
+
True,
|
|
46
|
+
"i8",
|
|
47
|
+
"benchmark.py",
|
|
48
|
+
val_size,
|
|
49
|
+
round(delta, 4),
|
|
50
|
+
round(val_size / delta),
|
|
51
|
+
round(val_size / delta / num_threads),
|
|
52
|
+
]
|
|
53
|
+
print(result)
|
|
54
|
+
result_list.append(result)
|
|
55
|
+
|
|
56
|
+
result_df = pd.DataFrame(
|
|
57
|
+
result_list,
|
|
58
|
+
columns=[
|
|
59
|
+
"iid_count",
|
|
60
|
+
"sid_count",
|
|
61
|
+
"num_threads",
|
|
62
|
+
"drive",
|
|
63
|
+
"include_error",
|
|
64
|
+
"release",
|
|
65
|
+
"dtype",
|
|
66
|
+
"source",
|
|
67
|
+
"val size",
|
|
68
|
+
"time",
|
|
69
|
+
"val per second",
|
|
70
|
+
"per thread",
|
|
71
|
+
],
|
|
72
|
+
)
|
|
73
|
+
return result_df
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def meta_test(
|
|
77
|
+
iid_count,
|
|
78
|
+
sid_start=5,
|
|
79
|
+
sid_end=None,
|
|
80
|
+
point_count=30,
|
|
81
|
+
drive_list=["ssd"],
|
|
82
|
+
plot_index=0,
|
|
83
|
+
):
|
|
84
|
+
# 5K vs 50K
|
|
85
|
+
# 50K vs 50K
|
|
86
|
+
# 500K vs 5K
|
|
87
|
+
if sid_end is None:
|
|
88
|
+
if iid_count <= 50_000:
|
|
89
|
+
sid_end = 50_000
|
|
90
|
+
else:
|
|
91
|
+
sid_end = 5_000
|
|
92
|
+
|
|
93
|
+
result = []
|
|
94
|
+
for sid_count in np.logspace(
|
|
95
|
+
np.log10(sid_start), np.log10(sid_end), point_count, base=10, dtype=int
|
|
96
|
+
):
|
|
97
|
+
for drive in drive_list:
|
|
98
|
+
for num_threads in [1, 12]:
|
|
99
|
+
result.append(
|
|
100
|
+
test_writes(iid_count, sid_count, num_threads, drive, False)
|
|
101
|
+
)
|
|
102
|
+
df = pd.concat(result)
|
|
103
|
+
df2 = df.pivot(
|
|
104
|
+
index="sid_count",
|
|
105
|
+
columns=["iid_count", "drive", "num_threads", "version"],
|
|
106
|
+
values="val per second",
|
|
107
|
+
)
|
|
108
|
+
df2.plot(marker=".", logx=True)
|
|
109
|
+
plt.savefig(
|
|
110
|
+
ssd_path
|
|
111
|
+
/ "plots"
|
|
112
|
+
/ f"plot{plot_index},iid_count{iid_count},{'_'.join(drive_list)}.png"
|
|
113
|
+
)
|
|
114
|
+
# plt.show()
|
|
115
|
+
return df
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
if __name__ == "__main__":
|
|
119
|
+
import matplotlib.pyplot as plt
|
|
120
|
+
|
|
121
|
+
plot_count = 0
|
|
122
|
+
for drive in ["ssd", "hdd"]:
|
|
123
|
+
for iid_count in [5_000, 50_000, 500_000]:
|
|
124
|
+
meta_test(
|
|
125
|
+
iid_count,
|
|
126
|
+
drive_list=[drive],
|
|
127
|
+
plot_index=plot_count,
|
|
128
|
+
)
|
|
129
|
+
plot_count += 1
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This is not really a bed file.
|
|
File without changes
|
|
Binary file
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
1 1:1:A:C 0.0 1 C A
|
|
2
|
+
1 1:2:A:C 0.0 2 C A
|
|
3
|
+
1 1:3:A:C 0.0 3 C A
|
|
4
|
+
1 1:4:A:C 0.0 4 C A
|
|
5
|
+
1 1:5:A:C 0.0 5 C A
|
|
6
|
+
1 1:6:A:C 0.0 6 C A
|
|
7
|
+
1 1:7:A:C 0.0 7 C A
|
|
8
|
+
1 1:8:A:C 0.0 8 C A
|
|
9
|
+
1 1:9:A:C 0.0 9 C A
|
|
10
|
+
1 1:10:A:C 0.0 10 C A
|
|
11
|
+
1 1:11:A:C 0.0 11 C A
|
|
12
|
+
1 1:12:A:C 0.0 12 C A
|
|
13
|
+
1 1:13:A:C 0.0 13 C A
|
|
14
|
+
1 1:14:A:C 0.0 14 C A
|
|
15
|
+
1 1:15:A:C 0.0 15 C A
|
|
16
|
+
1 1:16:A:C 0.0 16 C A
|
|
17
|
+
1 1:17:A:C 0.0 17 C A
|
|
18
|
+
1 1:18:A:C 0.0 18 C A
|
|
19
|
+
1 1:19:A:C 0.0 19 C A
|
|
20
|
+
1 1:20:A:C 0.0 20 C A
|
|
21
|
+
1 1:21:A:C 0.0 21 C A
|
|
22
|
+
1 1:22:A:C 0.0 22 C A
|
|
23
|
+
1 1:23:A:C 0.0 23 C A
|
|
24
|
+
1 1:24:A:C 0.0 24 C A
|
|
25
|
+
1 1:25:A:C 0.0 25 C A
|
|
26
|
+
1 1:26:A:C 0.0 26 C A
|
|
27
|
+
1 1:27:A:C 0.0 27 C A
|
|
28
|
+
1 1:28:A:C 0.0 28 C A
|
|
29
|
+
1 1:29:A:C 0.0 29 C A
|
|
30
|
+
1 1:30:A:C 0.0 30 C A
|
|
31
|
+
1 1:31:A:C 0.0 31 C A
|
|
32
|
+
1 1:32:A:C 0.0 32 C A
|
|
33
|
+
1 1:33:A:C 0.0 33 C A
|
|
34
|
+
1 1:34:A:C 0.0 34 C A
|
|
35
|
+
1 1:35:A:C 0.0 35 C A
|
|
36
|
+
1 1:36:A:C 0.0 36 C A
|
|
37
|
+
1 1:37:A:C 0.0 37 C A
|
|
38
|
+
1 1:38:A:C 0.0 38 C A
|
|
39
|
+
1 1:39:A:C 0.0 39 C A
|
|
40
|
+
1 1:40:A:C 0.0 40 C A
|
|
41
|
+
1 1:41:A:C 0.0 41 C A
|
|
42
|
+
1 1:42:A:C 0.0 42 C A
|
|
43
|
+
1 1:43:A:C 0.0 43 C A
|
|
44
|
+
1 1:44:A:C 0.0 44 C A
|
|
45
|
+
1 1:45:A:C 0.0 45 C A
|
|
46
|
+
1 1:46:A:C 0.0 46 C A
|
|
47
|
+
1 1:47:A:C 0.0 47 C A
|
|
48
|
+
1 1:48:A:C 0.0 48 C A
|
|
49
|
+
1 1:49:A:C 0.0 49 C A
|
|
50
|
+
1 1:50:A:C 0.0 50 C A
|
|
51
|
+
1 1:51:A:C 0.0 51 C A
|
|
52
|
+
1 1:52:A:C 0.0 52 C A
|
|
53
|
+
1 1:53:A:C 0.0 53 C A
|
|
54
|
+
1 1:54:A:C 0.0 54 C A
|
|
55
|
+
1 1:55:A:C 0.0 55 C A
|
|
56
|
+
1 1:56:A:C 0.0 56 C A
|
|
57
|
+
1 1:57:A:C 0.0 57 C A
|
|
58
|
+
1 1:58:A:C 0.0 58 C A
|
|
59
|
+
1 1:59:A:C 0.0 59 C A
|
|
60
|
+
1 1:60:A:C 0.0 60 C A
|
|
61
|
+
1 1:61:A:C 0.0 61 C A
|
|
62
|
+
1 1:62:A:C 0.0 62 C A
|
|
63
|
+
1 1:63:A:C 0.0 63 C A
|
|
64
|
+
1 1:64:A:C 0.0 64 C A
|
|
65
|
+
1 1:65:A:C 0.0 65 C A
|
|
66
|
+
1 1:66:A:C 0.0 66 C A
|
|
67
|
+
1 1:67:A:C 0.0 67 C A
|
|
68
|
+
1 1:68:A:C 0.0 68 C A
|
|
69
|
+
1 1:69:A:C 0.0 69 C A
|
|
70
|
+
1 1:70:A:C 0.0 70 C A
|
|
71
|
+
1 1:71:A:C 0.0 71 C A
|
|
72
|
+
1 1:72:A:C 0.0 72 C A
|
|
73
|
+
1 1:73:A:C 0.0 73 C A
|
|
74
|
+
1 1:74:A:C 0.0 74 C A
|
|
75
|
+
1 1:75:A:C 0.0 75 C A
|
|
76
|
+
1 1:76:A:C 0.0 76 C A
|
|
77
|
+
1 1:77:A:C 0.0 77 C A
|
|
78
|
+
1 1:78:A:C 0.0 78 C A
|
|
79
|
+
1 1:79:A:C 0.0 79 C A
|
|
80
|
+
1 1:80:A:C 0.0 80 C A
|
|
81
|
+
1 1:81:A:C 0.0 81 C A
|
|
82
|
+
1 1:82:A:C 0.0 82 C A
|
|
83
|
+
1 1:83:A:C 0.0 83 C A
|
|
84
|
+
1 1:84:A:C 0.0 84 C A
|
|
85
|
+
1 1:85:A:C 0.0 85 C A
|
|
86
|
+
1 1:86:A:C 0.0 86 C A
|
|
87
|
+
1 1:87:A:C 0.0 87 C A
|
|
88
|
+
1 1:88:A:C 0.0 88 C A
|
|
89
|
+
1 1:89:A:C 0.0 89 C A
|
|
90
|
+
1 1:90:A:C 0.0 90 C A
|
|
91
|
+
1 1:91:A:C 0.0 91 C A
|
|
92
|
+
1 1:92:A:C 0.0 92 C A
|
|
93
|
+
1 1:93:A:C 0.0 93 C A
|
|
94
|
+
1 1:94:A:C 0.0 94 C A
|
|
95
|
+
1 1:95:A:C 0.0 95 C A
|
|
96
|
+
1 1:96:A:C 0.0 96 C A
|
|
97
|
+
1 1:97:A:C 0.0 97 C A
|
|
98
|
+
1 1:98:A:C 0.0 98 C A
|
|
99
|
+
1 1:99:A:C 0.0 99 C A
|
|
100
|
+
1 1:100:A:C 0.0 100 C A
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
l2/3
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
l2/3
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
l2/3
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
l2/3
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
l2/
|
|
Binary file
|