multixem 0.3.2__tar.gz → 0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {multixem-0.3.2 → multixem-0.4}/MANIFEST.in +2 -1
- {multixem-0.3.2 → multixem-0.4}/PKG-INFO +27 -29
- {multixem-0.3.2 → multixem-0.4}/README.md +26 -28
- {multixem-0.3.2 → multixem-0.4}/multixem/__init__.py +1 -1
- {multixem-0.3.2 → multixem-0.4}/multixem/bootstrap_dataset.py +61 -40
- {multixem-0.3.2 → multixem-0.4}/multixem/bootstrap_maps.py +207 -154
- {multixem-0.3.2 → multixem-0.4}/multixem/bootstrap_statistics.py +18 -14
- {multixem-0.3.2 → multixem-0.4}/multixem/bootstrap_structures.py +45 -7
- {multixem-0.3.2 → multixem-0.4}/multixem/run.py +86 -52
- multixem-0.4/multixem/scripts/Rrot_sum_eigen.py +62 -0
- {multixem-0.3.2 → multixem-0.4}/multixem/tools.py +52 -1
- {multixem-0.3.2 → multixem-0.4}/multixem.egg-info/SOURCES.txt +1 -0
- {multixem-0.3.2 → multixem-0.4}/pyproject.toml +1 -1
- {multixem-0.3.2 → multixem-0.4}/LICENSE +0 -0
- {multixem-0.3.2 → multixem-0.4}/multixem/analyse_refinement.py +0 -0
- {multixem-0.3.2 → multixem-0.4}/multixem/data/Rrot_sum_eigen.json +0 -0
- {multixem-0.3.2 → multixem-0.4}/setup.cfg +0 -0
- {multixem-0.3.2 → multixem-0.4}/tests/test_argparse.py +0 -0
- {multixem-0.3.2 → multixem-0.4}/tests/test_integration.py +0 -0
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
include multixem/data/*.json
|
|
2
|
+
include multixem/scripts/*.py
|
|
3
|
+
|
|
2
4
|
prune dist
|
|
3
5
|
prune build
|
|
4
6
|
prune __venv__
|
|
@@ -11,7 +13,6 @@ prune docs_backup
|
|
|
11
13
|
prune scripts_backup
|
|
12
14
|
prune multixem_tmp
|
|
13
15
|
prune multixem_*
|
|
14
|
-
prune multixem/scripts
|
|
15
16
|
prune multixem/prototypes
|
|
16
17
|
|
|
17
18
|
global-exclude *.py[cod]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: multixem
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4
|
|
4
4
|
Summary: Refinement pipeline for multiple data sets in structure biology
|
|
5
5
|
Author-email: Martin Maly <martin.maly.mm@email.cz>
|
|
6
6
|
License: MIT
|
|
@@ -76,19 +76,18 @@ Example:
|
|
|
76
76
|
|
|
77
77
|
```bash
|
|
78
78
|
multixem bootstrap 10000 \
|
|
79
|
-
|
|
80
|
-
|
|
79
|
+
--hklin 1PGJ_data.mtz \
|
|
80
|
+
--model 1PGJ_model.pdb \
|
|
81
81
|
--hklin_free 1PGJ_data.mtz \
|
|
82
|
-
|
|
83
|
-
|
|
82
|
+
--prefix 1PGJ_bootstrap10000 \
|
|
83
|
+
--servalcat_args "--ncycle 10" \
|
|
84
84
|
--servalcat_confing "config.yaml" \
|
|
85
|
-
|
|
86
|
-
|
|
85
|
+
--n_bins 30 \
|
|
86
|
+
--n_proc 16 \
|
|
87
87
|
--geometry_cids 1PGJ_geometry_obj.txt \
|
|
88
|
-
--random_weights
|
|
89
88
|
```
|
|
90
89
|
|
|
91
|
-
|
|
90
|
+
The merged diffraction data (i.e. reflection) will be resampled with replacement 10,000 times. The subsequently refined structures will be analysed including calculation of mean, standard deviation, etc.
|
|
92
91
|
|
|
93
92
|
The file given in `1PGJ_geometry_obj.txt` defines parameters/features of the structure model under investigation. Each lines specifies an object using atomic CIDs divided by spaces. One CID in row denotes an occupancy, two an interactomic distance, three an angle between the atoms and four a torsion angle. For instance:
|
|
94
93
|
|
|
@@ -104,13 +103,12 @@ Unrestrained refinement of a small molecule can be performed using the following
|
|
|
104
103
|
|
|
105
104
|
```bash
|
|
106
105
|
multixem bootstrap 10000 \
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
--random_weights
|
|
106
|
+
--hklin 1513945.cif \
|
|
107
|
+
--model 1513945.cif \
|
|
108
|
+
--prefix 1513945_bootstrap10000 \
|
|
109
|
+
--servalcat_args "--unre --hydrogen yes --refine_h --adp aniso --no_solvent --adpr_weight 0 --ncycle 20" \
|
|
110
|
+
--n_bins 5 \
|
|
111
|
+
--n_proc 16 \
|
|
114
112
|
```
|
|
115
113
|
|
|
116
114
|
All currently available options are listed using:
|
|
@@ -134,25 +132,25 @@ Example: comparison of bovine, pork and human insulin:
|
|
|
134
132
|
|
|
135
133
|
```bash
|
|
136
134
|
multixem pipeline \
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
135
|
+
--hklin insuling_cow.mtz insuling_pig.mtz insuling_people.mtz \
|
|
136
|
+
--hklin_free insuling_people.mtz \
|
|
137
|
+
-p insulin \
|
|
138
|
+
--model insuling_cow.pdb insuling_pig.pdb insuling_people.pdb \
|
|
139
|
+
--n_bins 30 \
|
|
140
|
+
--n_proc 4 \
|
|
141
|
+
--unify_cell
|
|
144
142
|
```
|
|
145
143
|
|
|
146
144
|
Example: Merging and comparison of batches of unmerged diffraction data:
|
|
147
145
|
|
|
148
146
|
```bash
|
|
149
147
|
multixem pipeline \
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
148
|
+
--hklin_unmerged insuling_people_unmerged.mtz \
|
|
149
|
+
--hklin_free insuling_people.mtz \
|
|
150
|
+
-p insulin_people_600 \
|
|
151
|
+
--model insuling_people.pdb \
|
|
152
|
+
--n_bins 30 \
|
|
153
|
+
--n_proc 4 \
|
|
156
154
|
--n_batches 600
|
|
157
155
|
```
|
|
158
156
|
|
|
@@ -45,19 +45,18 @@ Example:
|
|
|
45
45
|
|
|
46
46
|
```bash
|
|
47
47
|
multixem bootstrap 10000 \
|
|
48
|
-
|
|
49
|
-
|
|
48
|
+
--hklin 1PGJ_data.mtz \
|
|
49
|
+
--model 1PGJ_model.pdb \
|
|
50
50
|
--hklin_free 1PGJ_data.mtz \
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
--prefix 1PGJ_bootstrap10000 \
|
|
52
|
+
--servalcat_args "--ncycle 10" \
|
|
53
53
|
--servalcat_confing "config.yaml" \
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
--n_bins 30 \
|
|
55
|
+
--n_proc 16 \
|
|
56
56
|
--geometry_cids 1PGJ_geometry_obj.txt \
|
|
57
|
-
--random_weights
|
|
58
57
|
```
|
|
59
58
|
|
|
60
|
-
|
|
59
|
+
The merged diffraction data (i.e. reflection) will be resampled with replacement 10,000 times. The subsequently refined structures will be analysed including calculation of mean, standard deviation, etc.
|
|
61
60
|
|
|
62
61
|
The file given in `1PGJ_geometry_obj.txt` defines parameters/features of the structure model under investigation. Each lines specifies an object using atomic CIDs divided by spaces. One CID in row denotes an occupancy, two an interactomic distance, three an angle between the atoms and four a torsion angle. For instance:
|
|
63
62
|
|
|
@@ -73,13 +72,12 @@ Unrestrained refinement of a small molecule can be performed using the following
|
|
|
73
72
|
|
|
74
73
|
```bash
|
|
75
74
|
multixem bootstrap 10000 \
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
--random_weights
|
|
75
|
+
--hklin 1513945.cif \
|
|
76
|
+
--model 1513945.cif \
|
|
77
|
+
--prefix 1513945_bootstrap10000 \
|
|
78
|
+
--servalcat_args "--unre --hydrogen yes --refine_h --adp aniso --no_solvent --adpr_weight 0 --ncycle 20" \
|
|
79
|
+
--n_bins 5 \
|
|
80
|
+
--n_proc 16 \
|
|
83
81
|
```
|
|
84
82
|
|
|
85
83
|
All currently available options are listed using:
|
|
@@ -103,25 +101,25 @@ Example: comparison of bovine, pork and human insulin:
|
|
|
103
101
|
|
|
104
102
|
```bash
|
|
105
103
|
multixem pipeline \
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
104
|
+
--hklin insuling_cow.mtz insuling_pig.mtz insuling_people.mtz \
|
|
105
|
+
--hklin_free insuling_people.mtz \
|
|
106
|
+
-p insulin \
|
|
107
|
+
--model insuling_cow.pdb insuling_pig.pdb insuling_people.pdb \
|
|
108
|
+
--n_bins 30 \
|
|
109
|
+
--n_proc 4 \
|
|
110
|
+
--unify_cell
|
|
113
111
|
```
|
|
114
112
|
|
|
115
113
|
Example: Merging and comparison of batches of unmerged diffraction data:
|
|
116
114
|
|
|
117
115
|
```bash
|
|
118
116
|
multixem pipeline \
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
117
|
+
--hklin_unmerged insuling_people_unmerged.mtz \
|
|
118
|
+
--hklin_free insuling_people.mtz \
|
|
119
|
+
-p insulin_people_600 \
|
|
120
|
+
--model insuling_people.pdb \
|
|
121
|
+
--n_bins 30 \
|
|
122
|
+
--n_proc 4 \
|
|
125
123
|
--n_batches 600
|
|
126
124
|
```
|
|
127
125
|
|
|
@@ -15,9 +15,10 @@ def bootstrap_dataset(
|
|
|
15
15
|
labin: str = "",
|
|
16
16
|
draw_factor: float = 1.0,
|
|
17
17
|
random_weights: bool = False,
|
|
18
|
+
dirichlet_alpha: float = 1.0,
|
|
18
19
|
col_free: str = "",
|
|
19
20
|
fraction_zero: float = 0.05,
|
|
20
|
-
):
|
|
21
|
+
) -> tuple[list[str], str]:
|
|
21
22
|
"""
|
|
22
23
|
Bootstrap the dataset from an MTZ file and save the results in new MTZ files.
|
|
23
24
|
|
|
@@ -38,14 +39,16 @@ def bootstrap_dataset(
|
|
|
38
39
|
fraction zero (float): Sets fraction of reflections with zero weight [0.0, 1.0)
|
|
39
40
|
Returns:
|
|
40
41
|
list of str: List of output MTZ filenames created during bootstrapping.
|
|
42
|
+
str: Path to the MTZ file without reflections without data
|
|
43
|
+
(or the original MTZ file if no filtering is applied).
|
|
41
44
|
"""
|
|
42
45
|
|
|
43
46
|
def resample(
|
|
44
47
|
n: int,
|
|
45
|
-
seed: int =
|
|
48
|
+
seed: int = 1000001,
|
|
46
49
|
draw_factor: float = 1.0,
|
|
47
50
|
column_name: str = "llweight",
|
|
48
|
-
):
|
|
51
|
+
) -> pandas.Series:
|
|
49
52
|
"""
|
|
50
53
|
Create a DataFrame`llweight` column using resampling with replacement.
|
|
51
54
|
|
|
@@ -74,41 +77,36 @@ def bootstrap_dataset(
|
|
|
74
77
|
|
|
75
78
|
return df_weight.rename(column_name)
|
|
76
79
|
|
|
77
|
-
def
|
|
80
|
+
def resample_dirichlet(
|
|
78
81
|
n: int,
|
|
79
82
|
zero_mask: numpy.ndarray,
|
|
80
|
-
|
|
83
|
+
dirichlet_alpha: float = 1.0,
|
|
84
|
+
seed: int = 1000001,
|
|
81
85
|
column_name: str = "llweight",
|
|
82
|
-
):
|
|
86
|
+
) -> pandas.Series:
|
|
83
87
|
"""
|
|
84
|
-
Create a DataFrame`llweight` column
|
|
85
|
-
a fraction of zero weights.
|
|
88
|
+
Create a DataFrame`llweight` column from random draws from uniform Dirichlet
|
|
89
|
+
distribution with alpha parameter while keeping a fraction of zero weights.
|
|
86
90
|
|
|
87
91
|
Args:
|
|
88
92
|
n (int): Number of items to resample.
|
|
89
|
-
seed (int): Random seed for reproducibility.
|
|
90
93
|
zero_mask (numpy.ndarray): Optional boolean mask;
|
|
91
94
|
rows set to True are forced to zero.
|
|
92
95
|
Applied if `fraction_zero` is zero.
|
|
96
|
+
dirichlet_alpha (float): Alpha parameter for the uniform Dirichlet
|
|
97
|
+
distribution. Must be positive. Default is 1.0.
|
|
98
|
+
seed (int): Random seed for reproducibility.
|
|
93
99
|
column_name (str): Name of the column to create in the DataFrame.
|
|
94
100
|
Returns:
|
|
95
101
|
pandas.Series: Series with the weights for each reflection.
|
|
96
102
|
"""
|
|
97
103
|
rng = numpy.random.default_rng(seed)
|
|
98
|
-
|
|
99
|
-
df_weight =
|
|
104
|
+
weights = rng.dirichlet(dirichlet_alpha * numpy.ones(n, dtype=float))
|
|
105
|
+
df_weight = pandas.Series(weights, index=range(n), name="index_resample")
|
|
100
106
|
# Set a fraction of weights to zero based on the provided mask
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
"Length mismatch between zero_mask and bin size in "
|
|
105
|
-
f"random resampling: {zero_mask_array.shape[0]} != {n}"
|
|
106
|
-
)
|
|
107
|
-
df_weight.loc[zero_mask_array] = 0.0
|
|
108
|
-
# Renormalize so that the total weight equals n
|
|
109
|
-
weight_sum = df_weight.sum()
|
|
110
|
-
assert weight_sum > 0
|
|
111
|
-
df_weight = df_weight * (n / weight_sum)
|
|
107
|
+
df_weight.loc[zero_mask] = 0.0
|
|
108
|
+
# Renormalize so that the sum of weights equals n
|
|
109
|
+
df_weight = df_weight * (n / df_weight.sum())
|
|
112
110
|
|
|
113
111
|
return df_weight.rename(column_name)
|
|
114
112
|
|
|
@@ -142,16 +140,24 @@ def bootstrap_dataset(
|
|
|
142
140
|
}
|
|
143
141
|
|
|
144
142
|
n_unique_orig = 0
|
|
145
|
-
|
|
146
|
-
# dropping reflections can cause problems, let's save the filtered dataset as MTZ
|
|
143
|
+
mtz_filtered_name = mtz_file
|
|
147
144
|
if labin and labin.split(",")[0] in df.columns:
|
|
148
|
-
df = df.dropna(subset=[labin.split(",")[0]])
|
|
149
|
-
# Save the filtered dataset as MTZ, preserving all original columns
|
|
150
|
-
mtz_filtered_name = (
|
|
151
|
-
f"{os.path.splitext(os.path.basename(mtz_file))[0]}_filtered.mtz"
|
|
152
|
-
)
|
|
153
|
-
write_mtz_from_df(df, mtz, columns=columns_dict, filename=mtz_filtered_name)
|
|
154
145
|
n_unique_orig = df.shape[0]
|
|
146
|
+
df = df.dropna(subset=[labin.split(",")[0]])
|
|
147
|
+
n_unique_after_drop = df.shape[0]
|
|
148
|
+
if n_unique_after_drop < n_unique_orig:
|
|
149
|
+
# Drop reflections without data and
|
|
150
|
+
# save the filtered dataset as MTZ, preserving all original columns
|
|
151
|
+
mtz_filtered_name = (
|
|
152
|
+
f"{os.path.splitext(os.path.basename(mtz_file))[0]}_filtered.mtz"
|
|
153
|
+
)
|
|
154
|
+
write_mtz_from_df(df, mtz, columns=columns_dict, filename=mtz_filtered_name)
|
|
155
|
+
logging.info(
|
|
156
|
+
f"Filtered {n_unique_orig - n_unique_after_drop} reflections"
|
|
157
|
+
f" with missing data in {labin} from {mtz_file}"
|
|
158
|
+
f" and saved to {mtz_filtered_name}"
|
|
159
|
+
)
|
|
160
|
+
n_unique_orig = n_unique_after_drop
|
|
155
161
|
else:
|
|
156
162
|
warnings.warn(
|
|
157
163
|
f"Column {labin} not found in MTZ file {mtz_file}. "
|
|
@@ -175,12 +181,27 @@ def bootstrap_dataset(
|
|
|
175
181
|
bins = [bin for _, bin in df.groupby("bin")]
|
|
176
182
|
|
|
177
183
|
# Create per-bin masks once and keep them for all bootstrap samples
|
|
178
|
-
zero_mask_bins = []
|
|
184
|
+
zero_mask_bins: list[numpy.ndarray] = []
|
|
179
185
|
if random_weights:
|
|
180
186
|
if abs(fraction_zero) > 1e-6:
|
|
181
|
-
# Create
|
|
187
|
+
# Create per-bin zero masks with an exact number of zero weights.
|
|
188
|
+
# This keeps the zero fraction deterministic for each bin while still
|
|
189
|
+
# being reproducible for a given seed.
|
|
182
190
|
rng = numpy.random.default_rng(seeds[0])
|
|
183
|
-
zero_mask_bins = [
|
|
191
|
+
zero_mask_bins = []
|
|
192
|
+
for bin in bins:
|
|
193
|
+
n_bin = len(bin)
|
|
194
|
+
n_zero = int(round(fraction_zero * n_bin))
|
|
195
|
+
if n_zero <= 0:
|
|
196
|
+
zero_mask_bins.append(numpy.zeros(n_bin, dtype=bool))
|
|
197
|
+
continue
|
|
198
|
+
if n_zero >= n_bin:
|
|
199
|
+
zero_mask_bins.append(numpy.ones(n_bin, dtype=bool))
|
|
200
|
+
continue
|
|
201
|
+
zero_idx = rng.choice(n_bin, size=n_zero, replace=False)
|
|
202
|
+
zero_mask = numpy.zeros(n_bin, dtype=bool)
|
|
203
|
+
zero_mask[zero_idx] = True
|
|
204
|
+
zero_mask_bins.append(zero_mask)
|
|
184
205
|
elif col_free:
|
|
185
206
|
# Use free reflections to create a mask for refls with zero weight
|
|
186
207
|
zero_mask_bins = [bin[col_free].eq(0).to_numpy() for bin in bins]
|
|
@@ -192,21 +213,21 @@ def bootstrap_dataset(
|
|
|
192
213
|
|
|
193
214
|
completeness_list = []
|
|
194
215
|
for i, seed in enumerate(seeds):
|
|
216
|
+
parts: list[pandas.Series] = []
|
|
195
217
|
if random_weights:
|
|
196
218
|
assert len(zero_mask_bins) == len(bins)
|
|
197
|
-
parts = []
|
|
198
219
|
for b, bin in enumerate(bins):
|
|
199
|
-
w =
|
|
220
|
+
w = resample_dirichlet(
|
|
200
221
|
len(bin),
|
|
201
222
|
zero_mask_bins[b],
|
|
202
|
-
|
|
223
|
+
dirichlet_alpha=dirichlet_alpha,
|
|
224
|
+
seed=seed + (b + 1) * 1000000,
|
|
203
225
|
)
|
|
204
226
|
parts.append(pandas.Series(w.values, index=bin.index, name="llweight"))
|
|
205
227
|
df_bootstrap1_weight = pandas.concat(parts).sort_index()
|
|
206
228
|
else:
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
w = resample(len(bin), seed, draw_factor)
|
|
229
|
+
for b, bin in enumerate(bins):
|
|
230
|
+
w = resample(len(bin), seed + (b + 1) * 1000000, draw_factor)
|
|
210
231
|
parts.append(pandas.Series(w.values, index=bin.index, name="llweight"))
|
|
211
232
|
df_bootstrap1_weight = pandas.concat(parts).sort_index()
|
|
212
233
|
|
|
@@ -307,4 +328,4 @@ def bootstrap_dataset(
|
|
|
307
328
|
completeness_orig = n_unique_orig / n_unique_expected
|
|
308
329
|
logging.info(f"Completeness of the original dataset: {completeness_orig:.2%}\n")
|
|
309
330
|
|
|
310
|
-
return mtzs_out
|
|
331
|
+
return mtzs_out, mtz_filtered_name
|