multixem 0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- multixem/__init__.py +6 -0
- multixem/analyse_refinement.py +656 -0
- multixem/bootstrap_dataset.py +310 -0
- multixem/bootstrap_maps.py +558 -0
- multixem/bootstrap_statistics.py +783 -0
- multixem/bootstrap_structures.py +1569 -0
- multixem/run.py +2193 -0
- multixem/tools.py +521 -0
- multixem-0.3.dist-info/METADATA +175 -0
- multixem-0.3.dist-info/RECORD +14 -0
- multixem-0.3.dist-info/WHEEL +5 -0
- multixem-0.3.dist-info/entry_points.txt +2 -0
- multixem-0.3.dist-info/licenses/LICENSE +373 -0
- multixem-0.3.dist-info/top_level.txt +1 -0
multixem/tools.py
ADDED
|
@@ -0,0 +1,521 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
import os
|
|
3
|
+
import numpy
|
|
4
|
+
import gemmi
|
|
5
|
+
import logging
|
|
6
|
+
import pandas
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def write_bin_stats(bin_stats_list, filename):
|
|
11
|
+
"""
|
|
12
|
+
Save bin statistics to a fixed-width text file.
|
|
13
|
+
|
|
14
|
+
Args:
|
|
15
|
+
bin_stats_list (list of dict): A list where each dictionary represents
|
|
16
|
+
statistics for a resolution bin. Each dictionary can have the
|
|
17
|
+
following keys:
|
|
18
|
+
- "bin" (int or str): Bin number or "overall" for overall statistics.
|
|
19
|
+
- "dmax" (float): Maximum resolution (Å).
|
|
20
|
+
- "dmin" (float): Minimum resolution (Å).
|
|
21
|
+
- "dmin_star2" (float): Minimum inverse resolution squared (Å⁻²).
|
|
22
|
+
- Additional keys may be included for other statistics.
|
|
23
|
+
filename (str): Path to the output file.
|
|
24
|
+
"""
|
|
25
|
+
# Convert to DataFrame and save
|
|
26
|
+
stats_df = pandas.DataFrame(bin_stats_list)
|
|
27
|
+
# stats_df.to_csv(stats_filename, index=False, sep="\t", float_format="%.4f")
|
|
28
|
+
# Round float columns to 4 decimal places
|
|
29
|
+
float_cols = stats_df.select_dtypes(include=["float"]).columns
|
|
30
|
+
stats_df[float_cols] = stats_df[float_cols].round(4)
|
|
31
|
+
# Save as fixed-width file
|
|
32
|
+
stats_df.to_string(
|
|
33
|
+
buf=open(filename, "w"),
|
|
34
|
+
index=False,
|
|
35
|
+
justify="right",
|
|
36
|
+
)
|
|
37
|
+
logging.info(f"Saved statistics to {filename}")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def write_mtz_from_df(df, mtz_ref, columns, filename):
|
|
41
|
+
"""
|
|
42
|
+
Create a gemmi.Mtz object from a pandas dataframe and save to file.
|
|
43
|
+
The numpy.float32 format is used for the data.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
df (pandas.DataFrame): DataFrame containing columns for H, K, L and other data.
|
|
47
|
+
mtz_ref (gemmi.Mtz): Reference MTZ file or object for cell and spacegroup.
|
|
48
|
+
columns (dict): Dictionary of column names and their MTZ data types
|
|
49
|
+
to include after H, K, L.
|
|
50
|
+
filename (str): Output filename for the MTZ file.
|
|
51
|
+
Returns:
|
|
52
|
+
None
|
|
53
|
+
"""
|
|
54
|
+
mtz = gemmi.Mtz(with_base=True)
|
|
55
|
+
if isinstance(mtz_ref, str):
|
|
56
|
+
mtz_ref = gemmi.read_mtz_file(mtz_ref)
|
|
57
|
+
mtz.spacegroup = mtz_ref.spacegroup
|
|
58
|
+
mtz.set_cell_for_all(mtz_ref.cell)
|
|
59
|
+
mtz.add_dataset(mtz_ref.datasets[0].dataset_name)
|
|
60
|
+
for col_name, col_type in columns.items():
|
|
61
|
+
mtz.add_column(col_name, col_type)
|
|
62
|
+
data = numpy.array(
|
|
63
|
+
df[["H", "K", "L"] + list(columns.keys())].values.astype(numpy.float64)
|
|
64
|
+
)
|
|
65
|
+
f32max = numpy.finfo(numpy.float32).max
|
|
66
|
+
data = numpy.clip(data, -f32max, f32max).astype(numpy.float32)
|
|
67
|
+
|
|
68
|
+
mtz.set_data(data)
|
|
69
|
+
mtz.write_to_file(filename)
|
|
70
|
+
logging.info(f"Saved {len(df)} reflections to {filename}")
|
|
71
|
+
return
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def calc_scale_real(df, column="FP", b=0, dmax=0.0, dmin=0.0):
|
|
75
|
+
"""
|
|
76
|
+
Assumes that df contains columns F1 and F2.
|
|
77
|
+
scale_real = sum_hkl (F1 * F2) / sum_hkl F2**2.
|
|
78
|
+
If denominator is zero, returns 1.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
df (pandas.DataFrame): DataFrame with columns for F1 and F2.
|
|
82
|
+
column (str): Base name of the columns for F1 and F2.
|
|
83
|
+
b (int): Bin number, used for warnings.
|
|
84
|
+
dmax (float): Maximum resolution for the bin, used in warnings.
|
|
85
|
+
dmin (float): Minimum resolution for the bin, used in warnings.
|
|
86
|
+
Returns:
|
|
87
|
+
float: Scale factor for the bin.
|
|
88
|
+
"""
|
|
89
|
+
nomin = (df[column + "1"] * df[column + "2"]).sum()
|
|
90
|
+
denomin = (df[column + "2"] ** 2).sum()
|
|
91
|
+
if not numpy.isclose(denomin, 0):
|
|
92
|
+
return nomin / denomin
|
|
93
|
+
else:
|
|
94
|
+
if b and dmax and dmin:
|
|
95
|
+
logging.warning(
|
|
96
|
+
f"Scale denominator for bin {b + 1} is zero"
|
|
97
|
+
f" ({dmax} - {dmin} A),"
|
|
98
|
+
" setting scale for this bin to 1."
|
|
99
|
+
)
|
|
100
|
+
return 1.0
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def calc_scale_complex(df, column="F_est", column_denom="", b=0, dmax=0.0, dmin=0.0):
|
|
104
|
+
"""
|
|
105
|
+
Assumes that df contains columns `F_est`1RE, `F_est`1IM,
|
|
106
|
+
`F_est`2RE, `F_est`2IM and `F_est`2.
|
|
107
|
+
scale_complex = sum_hkl (F1RE * F2RE + F1IM * F2IM) / sum_hkl F2**2
|
|
108
|
+
If denominator is zero, returns 1.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
df (pandas.DataFrame): DataFrame with columns for F1 and F2.
|
|
112
|
+
column (str): Base name of the columns for F1 and F2.
|
|
113
|
+
column_denom (str): Column name for the denominator, if different from F2.
|
|
114
|
+
b (int): Bin number, used for warnings.
|
|
115
|
+
dmax (float): Maximum resolution for the bin, used in warnings.
|
|
116
|
+
dmin (float): Minimum resolution for the bin, used in warnings.
|
|
117
|
+
Returns:
|
|
118
|
+
float: Scale factor for the bin.
|
|
119
|
+
"""
|
|
120
|
+
scale_complex_numer = (
|
|
121
|
+
(df[column + "1RE"] * df[column + "2RE"])
|
|
122
|
+
+ (df[column + "1IM"] * df[column + "2IM"])
|
|
123
|
+
).sum()
|
|
124
|
+
if column_denom:
|
|
125
|
+
scale_complex_denomin = (df[column_denom] ** 2).sum()
|
|
126
|
+
else:
|
|
127
|
+
scale_complex_denomin = (df[column + "2"] ** 2).sum()
|
|
128
|
+
# equivalent to the previous line:
|
|
129
|
+
# scale_complex_denomin = (df[column + '2RE']**2 + df[column + '2IM']**2).sum()
|
|
130
|
+
if not numpy.isclose(scale_complex_denomin, 0):
|
|
131
|
+
return scale_complex_numer / scale_complex_denomin
|
|
132
|
+
else:
|
|
133
|
+
logging.warning(
|
|
134
|
+
f"Scale denominator for bin {b + 1} is zero"
|
|
135
|
+
f" ({dmax} - {dmin} A),"
|
|
136
|
+
" setting scale for this bin to 1."
|
|
137
|
+
)
|
|
138
|
+
return 1.0
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def makeAddressStr(cra):
|
|
142
|
+
address = cra.chain.name + "/" + cra.residue.name + " "
|
|
143
|
+
address += str(cra.residue.seqid.num)
|
|
144
|
+
if cra.residue.seqid.icode.strip():
|
|
145
|
+
address += str(cra.residue.seqid.icode)
|
|
146
|
+
address += "/"
|
|
147
|
+
address += cra.atom.name
|
|
148
|
+
if cra.atom.has_altloc():
|
|
149
|
+
address += "."
|
|
150
|
+
address += cra.atom.altloc
|
|
151
|
+
return address
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
'''
|
|
155
|
+
def CRA2CID(cra):
|
|
156
|
+
"""Convert a gemmi.CRA to a CID string."""
|
|
157
|
+
cid = f"//{cra.chain.name}/{cra.residue.seqid.num}"
|
|
158
|
+
if cra.residue.seqid.icode.strip():
|
|
159
|
+
cid += f".{cra.residue.seqid.icode}"
|
|
160
|
+
cid += f"/{cra.atom.name}"
|
|
161
|
+
if cra.atom.has_altloc():
|
|
162
|
+
cid += f":{cra.atom.altloc}"
|
|
163
|
+
return cid
|
|
164
|
+
'''
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def CID2dict(cid, atom_level=True):
|
|
168
|
+
"""
|
|
169
|
+
Parse a CID string and extract chain, seqid, insertion code, atom name, and altloc.
|
|
170
|
+
|
|
171
|
+
Args:
|
|
172
|
+
cid (str): CID string in format //chain/seqnum[.inscode]/atom[:altloc][@symm]
|
|
173
|
+
|
|
174
|
+
Returns:
|
|
175
|
+
dict: Dictionary with keys chain, seqnum, inscode, atom, altloc, symm
|
|
176
|
+
"""
|
|
177
|
+
cid_no_symm = cid.split("@")[0]
|
|
178
|
+
cid_split = cid_no_symm.split("/")
|
|
179
|
+
assert len(cid_split) >= 2, f"CID select all the model: {cid}"
|
|
180
|
+
if atom_level:
|
|
181
|
+
assert len(cid_split) >= 5, f"Invalid CID for an atom: {cid}"
|
|
182
|
+
if len(cid.split("@")) == 2:
|
|
183
|
+
symm = cid.split("@")[1]
|
|
184
|
+
else:
|
|
185
|
+
symm = ""
|
|
186
|
+
if len(cid_split) >= 3:
|
|
187
|
+
chain = cid_split[2]
|
|
188
|
+
if len(cid_split) >= 4:
|
|
189
|
+
seqid_split = cid_split[3].split(".")
|
|
190
|
+
seqnum = seqid_split[0]
|
|
191
|
+
inscode = seqid_split[1] if len(seqid_split) > 1 else ""
|
|
192
|
+
if len(cid_split) >= 5:
|
|
193
|
+
atom_split = cid_split[4].split(":")
|
|
194
|
+
atom = atom_split[0]
|
|
195
|
+
altloc = atom_split[1] if len(atom_split) > 1 else ""
|
|
196
|
+
else:
|
|
197
|
+
atom = ""
|
|
198
|
+
altloc = ""
|
|
199
|
+
else:
|
|
200
|
+
seqnum = ""
|
|
201
|
+
inscode = ""
|
|
202
|
+
else:
|
|
203
|
+
chain = ""
|
|
204
|
+
|
|
205
|
+
return {
|
|
206
|
+
"chain": chain,
|
|
207
|
+
"seqnum": seqnum,
|
|
208
|
+
"inscode": inscode,
|
|
209
|
+
"atom": atom,
|
|
210
|
+
"altloc": altloc,
|
|
211
|
+
"symm": symm,
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def dict2CID(atom_dict):
|
|
216
|
+
"""Convert a dictionary with keys:
|
|
217
|
+
chain, seqnum, inscode, atom, altloc, symm
|
|
218
|
+
to a CID string."""
|
|
219
|
+
cid = f"//{atom_dict['chain']}/{atom_dict['seqnum']}"
|
|
220
|
+
if atom_dict["inscode"]:
|
|
221
|
+
cid += f".{atom_dict['inscode']}"
|
|
222
|
+
if atom_dict["atom"] or atom_dict["altloc"]:
|
|
223
|
+
cid += "/"
|
|
224
|
+
if atom_dict["atom"]:
|
|
225
|
+
cid += f"{atom_dict['atom']}"
|
|
226
|
+
else:
|
|
227
|
+
cid += "*"
|
|
228
|
+
if atom_dict["altloc"]:
|
|
229
|
+
cid += f":{atom_dict['altloc']}"
|
|
230
|
+
if atom_dict["symm"]:
|
|
231
|
+
cid += f"@{atom_dict['symm']}"
|
|
232
|
+
return cid
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def select_CIDs_of_residues(geometry_object):
|
|
236
|
+
"""
|
|
237
|
+
geometry_object is a dict with keys:
|
|
238
|
+
atom1, atom2, atom3, atom4 - CIDs of the atoms involved
|
|
239
|
+
|
|
240
|
+
Returns a list of CIDs with atom set to "*"
|
|
241
|
+
for the residues involved in the geometry object.
|
|
242
|
+
"""
|
|
243
|
+
|
|
244
|
+
cid_select_residues = []
|
|
245
|
+
for cid in [
|
|
246
|
+
geometry_object["atom1"],
|
|
247
|
+
geometry_object["atom2"],
|
|
248
|
+
geometry_object["atom3"],
|
|
249
|
+
geometry_object["atom4"],
|
|
250
|
+
]:
|
|
251
|
+
if not cid:
|
|
252
|
+
continue
|
|
253
|
+
atom_dict = CID2dict(cid, atom_level=True)
|
|
254
|
+
atom_dict["atom"] = "*"
|
|
255
|
+
cid_select_residue = dict2CID(atom_dict)
|
|
256
|
+
cid_select_residues.append(cid_select_residue)
|
|
257
|
+
|
|
258
|
+
return cid_select_residues
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
'''
|
|
262
|
+
def CID2RefmacRestraint(geometry_object):
|
|
263
|
+
"""geometry_object is a dict with keys:
|
|
264
|
+
atom1, atom2, atom3, atom4 - CIDs of the atoms involved
|
|
265
|
+
values - list of reference values, should contain one value only"""
|
|
266
|
+
|
|
267
|
+
ref_value = geometry_object.get("values", [None])[0]
|
|
268
|
+
refmacAddresses = []
|
|
269
|
+
for cid in [
|
|
270
|
+
geometry_object["atom1"],
|
|
271
|
+
geometry_object["atom2"],
|
|
272
|
+
geometry_object["atom3"],
|
|
273
|
+
geometry_object["atom4"],
|
|
274
|
+
]:
|
|
275
|
+
if not cid:
|
|
276
|
+
continue
|
|
277
|
+
atom_dict = CID2dict(cid, atom_level=True)
|
|
278
|
+
refmacAddress = f"chain {atom_dict['chain']} "
|
|
279
|
+
refmacAddress += f"resi {atom_dict['seqnum']} "
|
|
280
|
+
if atom_dict["inscode"]:
|
|
281
|
+
refmacAddress += f"inse {atom_dict['inscode']} "
|
|
282
|
+
refmacAddress += f"atom {atom_dict['atom']}"
|
|
283
|
+
if atom_dict["altloc"]:
|
|
284
|
+
refmacAddress += f" alte {atom_dict['altloc']}"
|
|
285
|
+
if atom_dict["symm"]:
|
|
286
|
+
refmacAddress += " symm y"
|
|
287
|
+
refmacAddresses.append(refmacAddress)
|
|
288
|
+
|
|
289
|
+
if not geometry_object["atom2"]:
|
|
290
|
+
# single atom in refmac syntax - may not be useful...
|
|
291
|
+
return refmacAddresses[0]
|
|
292
|
+
elif not geometry_object["atom3"]:
|
|
293
|
+
# external distance restraint
|
|
294
|
+
assert "@" not in geometry_object["atom1"], (
|
|
295
|
+
"For interatomic distances, the first atom must not be from"
|
|
296
|
+
f" a symmetry mate: {geometry_object['atom1']} {geometry_object['atom2']}"
|
|
297
|
+
)
|
|
298
|
+
if ref_value is None:
|
|
299
|
+
ref_value = 2.2
|
|
300
|
+
restraint = f"exte dist first {refmacAddresses[0]} second {refmacAddresses[1]}"
|
|
301
|
+
restraint += f" value {ref_value:<.2f} sigma 99 type 0"
|
|
302
|
+
elif not geometry_object["atom4"]:
|
|
303
|
+
# external angle restraint
|
|
304
|
+
assert "@" not in geometry_object["atom2"], (
|
|
305
|
+
"For angles, the second atom must not be from a symmetry mate:"
|
|
306
|
+
f" {geometry_object['atom1']} {geometry_object['atom2']}"
|
|
307
|
+
f" {geometry_object['atom3']}"
|
|
308
|
+
)
|
|
309
|
+
if ref_value is None:
|
|
310
|
+
ref_value = 120.0
|
|
311
|
+
restraint = f"exte angle first {refmacAddresses[0]} next {refmacAddresses[1]}"
|
|
312
|
+
restraint += f" next {refmacAddresses[2]}"
|
|
313
|
+
restraint += f" value {ref_value:<.2f} sigma 999 type 0"
|
|
314
|
+
else:
|
|
315
|
+
# external torsion restraint
|
|
316
|
+
if ref_value is None:
|
|
317
|
+
ref_value = 120.0
|
|
318
|
+
restraint = f"exte torsion first {refmacAddresses[0]} next {refmacAddresses[1]}"
|
|
319
|
+
restraint += f" next {refmacAddresses[2]} next {refmacAddresses[3]}"
|
|
320
|
+
restraint += f" value {ref_value:<.2f} sigma 999 type 0"
|
|
321
|
+
return restraint
|
|
322
|
+
'''
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def filename_replace_char(filename):
|
|
326
|
+
filename = filename.replace("=", "_equals_")
|
|
327
|
+
filename = filename.replace(">", "_gt_")
|
|
328
|
+
filename = filename.replace("<", "_lt_")
|
|
329
|
+
filename = re.sub(r"[^A-Za-z0-9_\-.]", "_", filename)
|
|
330
|
+
return filename
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def json_numpy_converter(o):
|
|
334
|
+
if isinstance(o, numpy.generic):
|
|
335
|
+
return o.item()
|
|
336
|
+
if isinstance(o, numpy.ndarray):
|
|
337
|
+
return o.tolist()
|
|
338
|
+
# Fallback: convert unknown objects to string
|
|
339
|
+
return str(o)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def scale_reflections(refl1, refl2, binner, bin_stats_list=[], output_mtz2_prefix=""):
|
|
343
|
+
"""
|
|
344
|
+
Scale reflections from refl2 to refl1 in resolution bins defined by binner.
|
|
345
|
+
|
|
346
|
+
Args:
|
|
347
|
+
refl1 (str or gemmi.Mtz or pandas.DataFrame): First reflection dataset.
|
|
348
|
+
refl2 (str or gemmi.Mtz or pandas.DataFrame): Second reflection dataset.
|
|
349
|
+
binner (gemmi.Binner): Binner object defining resolution bins.
|
|
350
|
+
bin_stats_list (list of dict): List to store statistics for each resolution bin.
|
|
351
|
+
output_mtz2_prefix (str): If provided, save scaled refl2 to MTZ file
|
|
352
|
+
with this prefix.
|
|
353
|
+
|
|
354
|
+
Returns:
|
|
355
|
+
df2_scaled (pandas.DataFrame): Scaled reflections from refl2.
|
|
356
|
+
bin_stats_list (list of dict): Updated list with statistics
|
|
357
|
+
for each resolution bin.
|
|
358
|
+
"""
|
|
359
|
+
|
|
360
|
+
if isinstance(refl1, str):
|
|
361
|
+
if not os.path.isfile(refl1):
|
|
362
|
+
raise FileNotFoundError(f"Reflection file not found: {refl1}")
|
|
363
|
+
mtz1 = gemmi.read_mtz_file(refl1)
|
|
364
|
+
df1 = pandas.DataFrame(data=mtz1.array, columns=mtz1.column_labels())
|
|
365
|
+
df1 = df1.astype({name: "int32" for name in ["H", "K", "L"]})
|
|
366
|
+
elif isinstance(refl1, gemmi.Mtz):
|
|
367
|
+
df1 = pandas.DataFrame(data=refl1.array, columns=refl1.column_labels())
|
|
368
|
+
df1 = df1.astype({name: "int32" for name in ["H", "K", "L"]})
|
|
369
|
+
else:
|
|
370
|
+
df1 = refl1
|
|
371
|
+
|
|
372
|
+
if isinstance(refl2, str):
|
|
373
|
+
if not os.path.isfile(refl2):
|
|
374
|
+
raise FileNotFoundError(f"Reflection file not found: {refl2}")
|
|
375
|
+
mtz2 = gemmi.read_mtz_file(refl2)
|
|
376
|
+
df2 = pandas.DataFrame(data=mtz2.array, columns=mtz2.column_labels())
|
|
377
|
+
df2 = df2.astype({name: "int32" for name in ["H", "K", "L"]})
|
|
378
|
+
elif isinstance(refl2, gemmi.Mtz):
|
|
379
|
+
df2 = pandas.DataFrame(data=refl2.array, columns=refl2.column_labels())
|
|
380
|
+
df2 = df2.astype({name: "int32" for name in ["H", "K", "L"]})
|
|
381
|
+
else:
|
|
382
|
+
df2 = refl2
|
|
383
|
+
|
|
384
|
+
f_col = "DELFWT" # Select only observed reflections
|
|
385
|
+
columns = ["FWT", "PHWT", "DELFWT", "PHDELWT"]
|
|
386
|
+
# afterwards, rename to FWT1, PHWT1, ..., FWT2, PHWT2, ...
|
|
387
|
+
# columns1 = [col + "1" for col in columns]
|
|
388
|
+
columns1_dict = {col: col + "1" for col in columns}
|
|
389
|
+
# columns2 = [col + "2" for col in columns]
|
|
390
|
+
columns2_dict = {col: col + "2" for col in columns}
|
|
391
|
+
|
|
392
|
+
df1 = df1[["H", "K", "L"] + columns] # Select only relevant columns
|
|
393
|
+
df1 = df1.dropna(subset=[f_col]) # Select only observed reflections
|
|
394
|
+
df1 = df1.rename(columns=columns1_dict) # Rename
|
|
395
|
+
# n_refl1 = len(df1)
|
|
396
|
+
# logging.info(f"No. unique reflections: {n_refl1} in file {mtz_file_1}")
|
|
397
|
+
|
|
398
|
+
llweight_col = ["llweight"] if "llweight" in df2.columns else []
|
|
399
|
+
df2 = df2[["H", "K", "L"] + columns + llweight_col]
|
|
400
|
+
df2 = df2.dropna(subset=[f_col]) # Select only observed reflections
|
|
401
|
+
if llweight_col:
|
|
402
|
+
df2 = df2.dropna(subset=llweight_col)
|
|
403
|
+
df2 = df2.rename(columns=columns2_dict)
|
|
404
|
+
# n_refl2 = len(df2)
|
|
405
|
+
# logging.info(f"No. unique reflections: {n_refl2} in file {mtz_file_2}")
|
|
406
|
+
|
|
407
|
+
# Extract common Miller indices (H, K, L)
|
|
408
|
+
df = pandas.merge(df1, df2, on=["H", "K", "L"])
|
|
409
|
+
# n_refl = len(df)
|
|
410
|
+
# logging.info(
|
|
411
|
+
# f"No. unique reflections: {n_refl} in common;"
|
|
412
|
+
# f" ratios to the originals: {n_refl / n_refl1:.4f} {n_refl / n_refl2:.4f}"
|
|
413
|
+
# )
|
|
414
|
+
hkl_common_array = numpy.array(df[["H", "K", "L"]].values, numpy.int32)
|
|
415
|
+
hkl_common_array = numpy.ascontiguousarray(hkl_common_array, dtype=numpy.int32)
|
|
416
|
+
# print(len(hkl_common_array)) # should be equal to n_refl
|
|
417
|
+
|
|
418
|
+
# Scaling per resolution bins
|
|
419
|
+
df["BIN"] = binner.get_bins(hkl_common_array)
|
|
420
|
+
|
|
421
|
+
df["FWT1RE"] = df["FWT1"] * numpy.cos(numpy.deg2rad(df["PHWT1"]))
|
|
422
|
+
df["FWT1IM"] = df["FWT1"] * numpy.sin(numpy.deg2rad(df["PHWT1"]))
|
|
423
|
+
df["FWT2RE"] = df["FWT2"] * numpy.cos(numpy.deg2rad(df["PHWT2"]))
|
|
424
|
+
df["FWT2IM"] = df["FWT2"] * numpy.sin(numpy.deg2rad(df["PHWT2"]))
|
|
425
|
+
df["DELFWT1RE"] = df["DELFWT1"] * numpy.cos(numpy.deg2rad(df["PHDELWT1"]))
|
|
426
|
+
df["DELFWT1IM"] = df["DELFWT1"] * numpy.sin(numpy.deg2rad(df["PHDELWT1"]))
|
|
427
|
+
df["DELFWT2RE"] = df["DELFWT2"] * numpy.cos(numpy.deg2rad(df["PHDELWT2"]))
|
|
428
|
+
df["DELFWT2IM"] = df["DELFWT2"] * numpy.sin(numpy.deg2rad(df["PHDELWT2"]))
|
|
429
|
+
|
|
430
|
+
if not bin_stats_list:
|
|
431
|
+
bin_stats_list = [
|
|
432
|
+
{
|
|
433
|
+
"bin": b + 1,
|
|
434
|
+
"dmax": binner.dmax_of_bin(b),
|
|
435
|
+
"dmin": binner.dmin_of_bin(b),
|
|
436
|
+
}
|
|
437
|
+
for b in range(binner.size)
|
|
438
|
+
]
|
|
439
|
+
if len(bin_stats_list) != binner.size:
|
|
440
|
+
logging.warning(
|
|
441
|
+
f"bin_stats_list has {len(bin_stats_list)} bins,"
|
|
442
|
+
f" but binner has {binner.size} bins.",
|
|
443
|
+
)
|
|
444
|
+
for b in range(len(bin_stats_list)):
|
|
445
|
+
df_bin = df[df["BIN"] == b]
|
|
446
|
+
|
|
447
|
+
scale_fwt = calc_scale_complex(
|
|
448
|
+
df_bin,
|
|
449
|
+
"FWT",
|
|
450
|
+
"FWT2",
|
|
451
|
+
b,
|
|
452
|
+
bin_stats_list[b]["dmax"],
|
|
453
|
+
bin_stats_list[b]["dmin"],
|
|
454
|
+
)
|
|
455
|
+
"""scale_delfwt = calc_scale_complex(
|
|
456
|
+
df_bin,
|
|
457
|
+
"DELFWT",
|
|
458
|
+
"DELFWT2",
|
|
459
|
+
b,
|
|
460
|
+
bin_stats_list[b]["dmax"],
|
|
461
|
+
bin_stats_list[b]["dmin"],
|
|
462
|
+
)"""
|
|
463
|
+
|
|
464
|
+
if len(df_bin) < 100:
|
|
465
|
+
logging.warning(
|
|
466
|
+
f"Less than 100 reflections in bin {b + 1}"
|
|
467
|
+
f" ({bin_stats_list[b]['dmax']:.4f} -"
|
|
468
|
+
f" {bin_stats_list[b]['dmin']:.4f} A)."
|
|
469
|
+
)
|
|
470
|
+
bin_stats_list[b]["scale_fwt"] = scale_fwt
|
|
471
|
+
bin_stats_list[b]["bin_count"] = len(df_bin)
|
|
472
|
+
fwt2_count = int(
|
|
473
|
+
pandas.to_numeric(df_bin["FWT2"], errors="coerce").notna().sum()
|
|
474
|
+
)
|
|
475
|
+
bin_stats_list[b]["fwt_count"] = fwt2_count
|
|
476
|
+
# bin_stats_list[b]["scale_delfwt"] = scale_delfwt
|
|
477
|
+
delfwt2_count = int(
|
|
478
|
+
pandas.to_numeric(df_bin["DELFWT2"], errors="coerce").notna().sum()
|
|
479
|
+
)
|
|
480
|
+
bin_stats_list[b]["delfwt_count"] = min(delfwt2_count, delfwt2_count)
|
|
481
|
+
|
|
482
|
+
# FWT
|
|
483
|
+
df.loc[df_bin.index, "FWT2SCRE"] = scale_fwt * df_bin["FWT2RE"]
|
|
484
|
+
df.loc[df_bin.index, "FWT2SCIM"] = scale_fwt * df_bin["FWT2IM"]
|
|
485
|
+
# DELFWT
|
|
486
|
+
df.loc[df_bin.index, "DELFWT2SCRE"] = scale_fwt * df_bin["DELFWT2RE"]
|
|
487
|
+
df.loc[df_bin.index, "DELFWT2SCIM"] = scale_fwt * df_bin["DELFWT2IM"]
|
|
488
|
+
|
|
489
|
+
df["FWT"] = numpy.hypot(
|
|
490
|
+
df["FWT2SCRE"].astype(numpy.float64),
|
|
491
|
+
df["FWT2SCIM"].astype(numpy.float64),
|
|
492
|
+
)
|
|
493
|
+
df["PHWT"] = numpy.rad2deg(numpy.arctan2(df["FWT2SCIM"], df["FWT2SCRE"]))
|
|
494
|
+
df["DELFWT"] = numpy.hypot(
|
|
495
|
+
df["DELFWT2SCRE"].astype(numpy.float64),
|
|
496
|
+
df["DELFWT2SCIM"].astype(numpy.float64),
|
|
497
|
+
)
|
|
498
|
+
df["PHDELWT"] = numpy.rad2deg(numpy.arctan2(df["DELFWT2SCIM"], df["DELFWT2SCRE"]))
|
|
499
|
+
df2_scaled = df[
|
|
500
|
+
["H", "K", "L", "FWT", "PHWT", "DELFWT", "PHDELWT"] + llweight_col
|
|
501
|
+
].copy()
|
|
502
|
+
|
|
503
|
+
if output_mtz2_prefix:
|
|
504
|
+
output_mtz2 = f"{output_mtz2_prefix}_scaled.mtz"
|
|
505
|
+
columns_to_write_list = [
|
|
506
|
+
"FWT",
|
|
507
|
+
"PHWT",
|
|
508
|
+
"DELFWT",
|
|
509
|
+
"PHDELWT",
|
|
510
|
+
]
|
|
511
|
+
columns_to_write_dict = {
|
|
512
|
+
col: ("F" if not col.startswith("PH") else "P")
|
|
513
|
+
for col in columns_to_write_list
|
|
514
|
+
}
|
|
515
|
+
if llweight_col:
|
|
516
|
+
columns_to_write_dict["llweight"] = "I"
|
|
517
|
+
write_mtz_from_df(df, mtz1, columns_to_write_dict, output_mtz2)
|
|
518
|
+
stats_filename = f"{output_mtz2_prefix}_scaled_bin_stats.txt"
|
|
519
|
+
write_bin_stats(bin_stats_list, stats_filename)
|
|
520
|
+
|
|
521
|
+
return df2_scaled, bin_stats_list
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: multixem
|
|
3
|
+
Version: 0.3
|
|
4
|
+
Summary: Refinement pipeline for multiple data sets in structure biology
|
|
5
|
+
Author-email: Martin Maly <martin.maly.mm@email.cz>
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: <4.0,>=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: pandas>=1.1.0
|
|
11
|
+
Requires-Dist: gemmi>=0.6.2
|
|
12
|
+
Requires-Dist: numpy>=1.15
|
|
13
|
+
Requires-Dist: matplotlib>=3.0.0
|
|
14
|
+
Requires-Dist: pyyaml>=6.0
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: vulture>=1.0; extra == "dev"
|
|
17
|
+
Requires-Dist: bandit>=1.6; extra == "dev"
|
|
18
|
+
Requires-Dist: safety>=1.8; extra == "dev"
|
|
19
|
+
Requires-Dist: isort>=5.3; extra == "dev"
|
|
20
|
+
Requires-Dist: flake8-bugbear>=19.8; extra == "dev"
|
|
21
|
+
Requires-Dist: black>=18.3a0; extra == "dev"
|
|
22
|
+
Requires-Dist: mypy>=0.730.0; extra == "dev"
|
|
23
|
+
Requires-Dist: ipython>=7.7; extra == "dev"
|
|
24
|
+
Requires-Dist: pytest>=5.0; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-cov>=2.7; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-mock>=1.10; extra == "dev"
|
|
27
|
+
Requires-Dist: pep8-naming>=0.8.2; extra == "dev"
|
|
28
|
+
Requires-Dist: portray>=1.3.0; extra == "dev"
|
|
29
|
+
Requires-Dist: cruft>=2.2; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
**Multixem** Refinement pipeline for multiple data sets in structure biology
|
|
33
|
+
_________________
|
|
34
|
+
|
|
35
|
+
[](http://badge.fury.io/py/multixem)
|
|
36
|
+
[](https://github.com/MartinMalyMM/multixem/actions?query=workflow%3ATest)
|
|
37
|
+
[](https://pepy.tech/project/multixem)
|
|
38
|
+
[](https://github.com/psf/black)
|
|
39
|
+
_________________
|
|
40
|
+
|
|
41
|
+
*This is still a work in progress.*
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
Installation
|
|
45
|
+
============
|
|
46
|
+
|
|
47
|
+
Install the latest code from this GitHub repository using:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pip install git+https://github.com/MartinMalyMM/multixem.git
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Or install the released package from PyPI with `pip`:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
pip install multixem
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
It is recommended to use a Python virtual environment:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python -m venv .venv
|
|
63
|
+
source .venv/bin/activate
|
|
64
|
+
pip install multixem # or pip install git+https://github.com/MartinMalyMM/multixem.git
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Most of the dependencies (NumPy, pandas, [GEMMI](https://github.com/project-gemmi/gemmi), MatPlotLib Python modules) will be installed automatically. You will also need [Servalcat](https://github.com/kyamashita/servalcat) installed, typically from a recent `CCP4` installation which also includes the [Monomer Library](https://github.com/MonomerLibrary/monomers).
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
Bootstrap
|
|
71
|
+
=========
|
|
72
|
+
|
|
73
|
+
Subcommand `bootstrap`: Perform the bootstrap protocol - run multiple refinements in parallel against resampled sub data sets.
|
|
74
|
+
|
|
75
|
+
Example:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
multixem bootstrap 10000 \
|
|
79
|
+
--hklin 1PGJ_data.mtz \
|
|
80
|
+
--model 1PGJ_model.pdb \
|
|
81
|
+
--hklin_free 1PGJ_data.mtz \
|
|
82
|
+
--prefix 1PGJ_bootstrap10000 \
|
|
83
|
+
--servalcat_args "--ncycle 10" \
|
|
84
|
+
--servalcat_confing "config.yaml" \
|
|
85
|
+
--n_bins 30 \
|
|
86
|
+
--n_proc 16 \
|
|
87
|
+
--geometry_cids 1PGJ_geometry_obj.txt \
|
|
88
|
+
--random_weights
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Random weights will be assigned to reflection likelihood terms in the target log-likelihood function. A fraction of reflection will be assigned zero weights based on the provided free flag in --hklin_free or --hklin, or randomly if no free flag is provided, controlled by --fraction_zero.
|
|
92
|
+
|
|
93
|
+
The file given in `1PGJ_geometry_obj.txt` defines parameters/features of the structure model under investigation. Each lines specifies an object using atomic CIDs divided by spaces. One CID in row denotes an occupancy, two an interactomic distance, three an angle between the atoms and four a torsion angle. For instance:
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
//A/505/O2
|
|
97
|
+
//A/505/O2 //A/262/CG
|
|
98
|
+
//A/505/O2 //A/262/CG //A/262/CD
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Regarding the distances and angles, the procedure is suitable for investigation non-covalently linked atoms. The environment is locally unrestrained (Van der Waals anti-bumping restraints switched off) to provide unbiased results.
|
|
102
|
+
|
|
103
|
+
Unrestrained refinement of a small molecule can be performed using the following setting:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
multixem bootstrap 10000 \
|
|
107
|
+
--hklin 1513945.cif \
|
|
108
|
+
--model 1513945.cif \
|
|
109
|
+
--prefix 1513945_bootstrap10000 \
|
|
110
|
+
--servalcat_args "--unre --hydrogen yes --refine_h --adp aniso --no_solvent --adpr_weight 0 --ncycle 20" \
|
|
111
|
+
--n_bins 20 \
|
|
112
|
+
--n_proc 16 \
|
|
113
|
+
--random_weights
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
All currently available options are listed using:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
multixem bootstrap --help
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
The options for Servalcat which could be provided through the `--servalcat_args` parameter to this *Multixem* pipeline can be listed using:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
servalcat refine_xtal_norefmac --help
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Comparison of isomorphous data sets
|
|
129
|
+
===================================
|
|
130
|
+
|
|
131
|
+
Subcommand `pipeline`: Compare the given diffraction data sets, refine given structure model(s) against them and compare them including calculation of isomorphous difference density maps (|Fobs,n|e^iɸn – k|Fobs,1|e^iɸ1). The input data set should be cut at the same resolution.
|
|
132
|
+
|
|
133
|
+
Example: comparison of bovine, pork and human insulin:
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
multixem pipeline \
|
|
137
|
+
--hklin insuling_cow.mtz insuling_pig.mtz insuling_people.mtz \
|
|
138
|
+
--hklin_free insuling_people.mtz \
|
|
139
|
+
-p insulin \
|
|
140
|
+
--model insuling_cow.pdb insuling_pig.pdb insuling_people.pdb \
|
|
141
|
+
--n_bins 30 \
|
|
142
|
+
--n_proc 4 \
|
|
143
|
+
--unify_cell
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Example: Merging and comparison of batches of unmerged diffraction data:
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
multixem pipeline \
|
|
150
|
+
--hklin_unmerged insuling_people_unmerged.mtz \
|
|
151
|
+
--hklin_free insuling_people.mtz \
|
|
152
|
+
-p insulin_people_600 \
|
|
153
|
+
--model insuling_people.pdb \
|
|
154
|
+
--n_bins 30 \
|
|
155
|
+
--n_proc 4 \
|
|
156
|
+
--n_batches 600
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
All currently available options are listed using:
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
multixem pipeline --help
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Tests
|
|
167
|
+
=====
|
|
168
|
+
|
|
169
|
+
Automatic tests are implemented using `pytest`. They can be run using the following command:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
python -m pytest -s -v
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
The integration tests use data from 6-phosphogluconate dehydrogenase (PDB [1PGJ](https://www.rcsb.org/structure/1PGJ), [manuscript](https://doi.org/10.1006/jmbi.1998.2059)) and human, porcine, and bovine insulin ([manuscript](https://doi.org/10.1107/S2059798325004589), [raw data (DOI 10.5281/zenodo.13890874)](https://doi.org/10.5281/zenodo.13890874)).
|