multixem 0.3.3__tar.gz → 0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: multixem
3
- Version: 0.3.3
3
+ Version: 0.4
4
4
  Summary: Refinement pipeline for multiple data sets in structure biology
5
5
  Author-email: Martin Maly <martin.maly.mm@email.cz>
6
6
  License: MIT
@@ -76,19 +76,18 @@ Example:
76
76
 
77
77
  ```bash
78
78
  multixem bootstrap 10000 \
79
- --hklin 1PGJ_data.mtz \
80
- --model 1PGJ_model.pdb \
79
+ --hklin 1PGJ_data.mtz \
80
+ --model 1PGJ_model.pdb \
81
81
  --hklin_free 1PGJ_data.mtz \
82
- --prefix 1PGJ_bootstrap10000 \
83
- --servalcat_args "--ncycle 10" \
82
+ --prefix 1PGJ_bootstrap10000 \
83
+ --servalcat_args "--ncycle 10" \
84
84
  --servalcat_confing "config.yaml" \
85
- --n_bins 30 \
86
- --n_proc 16 \
85
+ --n_bins 30 \
86
+ --n_proc 16 \
87
87
  --geometry_cids 1PGJ_geometry_obj.txt \
88
- --random_weights
89
88
  ```
90
89
 
91
- Random weights will be assigned to reflection likelihood terms in the target log-likelihood function. A fraction of reflection will be assigned zero weights based on the provided free flag in --hklin_free or --hklin, or randomly if no free flag is provided, controlled by --fraction_zero.
90
+ The merged diffraction data (i.e. reflection) will be resampled with replacement 10,000 times. The subsequently refined structures will be analysed including calculation of mean, standard deviation, etc.
92
91
 
93
92
  The file given in `1PGJ_geometry_obj.txt` defines parameters/features of the structure model under investigation. Each lines specifies an object using atomic CIDs divided by spaces. One CID in row denotes an occupancy, two an interactomic distance, three an angle between the atoms and four a torsion angle. For instance:
94
93
 
@@ -104,13 +103,12 @@ Unrestrained refinement of a small molecule can be performed using the following
104
103
 
105
104
  ```bash
106
105
  multixem bootstrap 10000 \
107
- --hklin 1513945.cif \
108
- --model 1513945.cif \
109
- --prefix 1513945_bootstrap10000 \
110
- --servalcat_args "--unre --hydrogen yes --refine_h --adp aniso --no_solvent --adpr_weight 0 --ncycle 20" \
111
- --n_bins 20 \
112
- --n_proc 16 \
113
- --random_weights
106
+ --hklin 1513945.cif \
107
+ --model 1513945.cif \
108
+ --prefix 1513945_bootstrap10000 \
109
+ --servalcat_args "--unre --hydrogen yes --refine_h --adp aniso --no_solvent --adpr_weight 0 --ncycle 20" \
110
+ --n_bins 5 \
111
+ --n_proc 16 \
114
112
  ```
115
113
 
116
114
  All currently available options are listed using:
@@ -134,25 +132,25 @@ Example: comparison of bovine, pork and human insulin:
134
132
 
135
133
  ```bash
136
134
  multixem pipeline \
137
- --hklin insuling_cow.mtz insuling_pig.mtz insuling_people.mtz \
138
- --hklin_free insuling_people.mtz \
139
- -p insulin \
140
- --model insuling_cow.pdb insuling_pig.pdb insuling_people.pdb \
141
- --n_bins 30 \
142
- --n_proc 4 \
143
- --unify_cell
135
+ --hklin insuling_cow.mtz insuling_pig.mtz insuling_people.mtz \
136
+ --hklin_free insuling_people.mtz \
137
+ -p insulin \
138
+ --model insuling_cow.pdb insuling_pig.pdb insuling_people.pdb \
139
+ --n_bins 30 \
140
+ --n_proc 4 \
141
+ --unify_cell
144
142
  ```
145
143
 
146
144
  Example: Merging and comparison of batches of unmerged diffraction data:
147
145
 
148
146
  ```bash
149
147
  multixem pipeline \
150
- --hklin_unmerged insuling_people_unmerged.mtz \
151
- --hklin_free insuling_people.mtz \
152
- -p insulin_people_600 \
153
- --model insuling_people.pdb \
154
- --n_bins 30 \
155
- --n_proc 4 \
148
+ --hklin_unmerged insuling_people_unmerged.mtz \
149
+ --hklin_free insuling_people.mtz \
150
+ -p insulin_people_600 \
151
+ --model insuling_people.pdb \
152
+ --n_bins 30 \
153
+ --n_proc 4 \
156
154
  --n_batches 600
157
155
  ```
158
156
 
@@ -45,19 +45,18 @@ Example:
45
45
 
46
46
  ```bash
47
47
  multixem bootstrap 10000 \
48
- --hklin 1PGJ_data.mtz \
49
- --model 1PGJ_model.pdb \
48
+ --hklin 1PGJ_data.mtz \
49
+ --model 1PGJ_model.pdb \
50
50
  --hklin_free 1PGJ_data.mtz \
51
- --prefix 1PGJ_bootstrap10000 \
52
- --servalcat_args "--ncycle 10" \
51
+ --prefix 1PGJ_bootstrap10000 \
52
+ --servalcat_args "--ncycle 10" \
53
53
  --servalcat_confing "config.yaml" \
54
- --n_bins 30 \
55
- --n_proc 16 \
54
+ --n_bins 30 \
55
+ --n_proc 16 \
56
56
  --geometry_cids 1PGJ_geometry_obj.txt \
57
- --random_weights
58
57
  ```
59
58
 
60
- Random weights will be assigned to reflection likelihood terms in the target log-likelihood function. A fraction of reflection will be assigned zero weights based on the provided free flag in --hklin_free or --hklin, or randomly if no free flag is provided, controlled by --fraction_zero.
59
+ The merged diffraction data (i.e. reflection) will be resampled with replacement 10,000 times. The subsequently refined structures will be analysed including calculation of mean, standard deviation, etc.
61
60
 
62
61
  The file given in `1PGJ_geometry_obj.txt` defines parameters/features of the structure model under investigation. Each lines specifies an object using atomic CIDs divided by spaces. One CID in row denotes an occupancy, two an interactomic distance, three an angle between the atoms and four a torsion angle. For instance:
63
62
 
@@ -73,13 +72,12 @@ Unrestrained refinement of a small molecule can be performed using the following
73
72
 
74
73
  ```bash
75
74
  multixem bootstrap 10000 \
76
- --hklin 1513945.cif \
77
- --model 1513945.cif \
78
- --prefix 1513945_bootstrap10000 \
79
- --servalcat_args "--unre --hydrogen yes --refine_h --adp aniso --no_solvent --adpr_weight 0 --ncycle 20" \
80
- --n_bins 20 \
81
- --n_proc 16 \
82
- --random_weights
75
+ --hklin 1513945.cif \
76
+ --model 1513945.cif \
77
+ --prefix 1513945_bootstrap10000 \
78
+ --servalcat_args "--unre --hydrogen yes --refine_h --adp aniso --no_solvent --adpr_weight 0 --ncycle 20" \
79
+ --n_bins 5 \
80
+ --n_proc 16 \
83
81
  ```
84
82
 
85
83
  All currently available options are listed using:
@@ -103,25 +101,25 @@ Example: comparison of bovine, pork and human insulin:
103
101
 
104
102
  ```bash
105
103
  multixem pipeline \
106
- --hklin insuling_cow.mtz insuling_pig.mtz insuling_people.mtz \
107
- --hklin_free insuling_people.mtz \
108
- -p insulin \
109
- --model insuling_cow.pdb insuling_pig.pdb insuling_people.pdb \
110
- --n_bins 30 \
111
- --n_proc 4 \
112
- --unify_cell
104
+ --hklin insuling_cow.mtz insuling_pig.mtz insuling_people.mtz \
105
+ --hklin_free insuling_people.mtz \
106
+ -p insulin \
107
+ --model insuling_cow.pdb insuling_pig.pdb insuling_people.pdb \
108
+ --n_bins 30 \
109
+ --n_proc 4 \
110
+ --unify_cell
113
111
  ```
114
112
 
115
113
  Example: Merging and comparison of batches of unmerged diffraction data:
116
114
 
117
115
  ```bash
118
116
  multixem pipeline \
119
- --hklin_unmerged insuling_people_unmerged.mtz \
120
- --hklin_free insuling_people.mtz \
121
- -p insulin_people_600 \
122
- --model insuling_people.pdb \
123
- --n_bins 30 \
124
- --n_proc 4 \
117
+ --hklin_unmerged insuling_people_unmerged.mtz \
118
+ --hklin_free insuling_people.mtz \
119
+ -p insulin_people_600 \
120
+ --model insuling_people.pdb \
121
+ --n_bins 30 \
122
+ --n_proc 4 \
125
123
  --n_batches 600
126
124
  ```
127
125
 
@@ -3,4 +3,4 @@
3
3
 
4
4
  Refinement pipeline for multiple data sets in structure biology
5
5
  """
6
- __version__ = "0.3.3"
6
+ __version__ = "0.4"
@@ -15,9 +15,10 @@ def bootstrap_dataset(
15
15
  labin: str = "",
16
16
  draw_factor: float = 1.0,
17
17
  random_weights: bool = False,
18
+ dirichlet_alpha: float = 1.0,
18
19
  col_free: str = "",
19
20
  fraction_zero: float = 0.05,
20
- ):
21
+ ) -> tuple[list[str], str]:
21
22
  """
22
23
  Bootstrap the dataset from an MTZ file and save the results in new MTZ files.
23
24
 
@@ -38,14 +39,16 @@ def bootstrap_dataset(
38
39
  fraction zero (float): Sets fraction of reflections with zero weight [0.0, 1.0)
39
40
  Returns:
40
41
  list of str: List of output MTZ filenames created during bootstrapping.
42
+ str: Path to the MTZ file without reflections without data
43
+ (or the original MTZ file if no filtering is applied).
41
44
  """
42
45
 
43
46
  def resample(
44
47
  n: int,
45
- seed: int = 1001,
48
+ seed: int = 1000001,
46
49
  draw_factor: float = 1.0,
47
50
  column_name: str = "llweight",
48
- ):
51
+ ) -> pandas.Series:
49
52
  """
50
53
  Create a DataFrame`llweight` column using resampling with replacement.
51
54
 
@@ -74,41 +77,36 @@ def bootstrap_dataset(
74
77
 
75
78
  return df_weight.rename(column_name)
76
79
 
77
- def resample_random(
80
+ def resample_dirichlet(
78
81
  n: int,
79
82
  zero_mask: numpy.ndarray,
80
- seed: int = 1001,
83
+ dirichlet_alpha: float = 1.0,
84
+ seed: int = 1000001,
81
85
  column_name: str = "llweight",
82
- ):
86
+ ) -> pandas.Series:
83
87
  """
84
- Create a DataFrame`llweight` column using random resampling and keeping
85
- a fraction of zero weights.
88
+ Create a DataFrame`llweight` column from random draws from uniform Dirichlet
89
+ distribution with alpha parameter while keeping a fraction of zero weights.
86
90
 
87
91
  Args:
88
92
  n (int): Number of items to resample.
89
- seed (int): Random seed for reproducibility.
90
93
  zero_mask (numpy.ndarray): Optional boolean mask;
91
94
  rows set to True are forced to zero.
92
95
  Applied if `fraction_zero` is zero.
96
+ dirichlet_alpha (float): Alpha parameter for the uniform Dirichlet
97
+ distribution. Must be positive. Default is 1.0.
98
+ seed (int): Random seed for reproducibility.
93
99
  column_name (str): Name of the column to create in the DataFrame.
94
100
  Returns:
95
101
  pandas.Series: Series with the weights for each reflection.
96
102
  """
97
103
  rng = numpy.random.default_rng(seed)
98
- df_random = pandas.DataFrame(rng.random(size=n), columns=["index_resample"])
99
- df_weight = df_random["index_resample"].copy()
104
+ weights = rng.dirichlet(dirichlet_alpha * numpy.ones(n, dtype=float))
105
+ df_weight = pandas.Series(weights, index=range(n), name="index_resample")
100
106
  # Set a fraction of weights to zero based on the provided mask
101
- zero_mask_array = numpy.asarray(zero_mask, dtype=bool)
102
- if zero_mask_array.shape[0] != n:
103
- raise ValueError(
104
- "Length mismatch between zero_mask and bin size in "
105
- f"random resampling: {zero_mask_array.shape[0]} != {n}"
106
- )
107
- df_weight.loc[zero_mask_array] = 0.0
108
- # Renormalize so that the total weight equals n
109
- weight_sum = df_weight.sum()
110
- assert weight_sum > 0
111
- df_weight = df_weight * (n / weight_sum)
107
+ df_weight.loc[zero_mask] = 0.0
108
+ # Renormalize so that the sum of weights equals n
109
+ df_weight = df_weight * (n / df_weight.sum())
112
110
 
113
111
  return df_weight.rename(column_name)
114
112
 
@@ -142,16 +140,24 @@ def bootstrap_dataset(
142
140
  }
143
141
 
144
142
  n_unique_orig = 0
145
- # i_col = "IMEAN" # can be just "I" after servalcat fw or sigmaa, or IMEAN?
146
- # dropping reflections can cause problems, let's save the filtered dataset as MTZ
143
+ mtz_filtered_name = mtz_file
147
144
  if labin and labin.split(",")[0] in df.columns:
148
- df = df.dropna(subset=[labin.split(",")[0]])
149
- # Save the filtered dataset as MTZ, preserving all original columns
150
- mtz_filtered_name = (
151
- f"{os.path.splitext(os.path.basename(mtz_file))[0]}_filtered.mtz"
152
- )
153
- write_mtz_from_df(df, mtz, columns=columns_dict, filename=mtz_filtered_name)
154
145
  n_unique_orig = df.shape[0]
146
+ df = df.dropna(subset=[labin.split(",")[0]])
147
+ n_unique_after_drop = df.shape[0]
148
+ if n_unique_after_drop < n_unique_orig:
149
+ # Drop reflections without data and
150
+ # save the filtered dataset as MTZ, preserving all original columns
151
+ mtz_filtered_name = (
152
+ f"{os.path.splitext(os.path.basename(mtz_file))[0]}_filtered.mtz"
153
+ )
154
+ write_mtz_from_df(df, mtz, columns=columns_dict, filename=mtz_filtered_name)
155
+ logging.info(
156
+ f"Filtered {n_unique_orig - n_unique_after_drop} reflections"
157
+ f" with missing data in {labin} from {mtz_file}"
158
+ f" and saved to {mtz_filtered_name}"
159
+ )
160
+ n_unique_orig = n_unique_after_drop
155
161
  else:
156
162
  warnings.warn(
157
163
  f"Column {labin} not found in MTZ file {mtz_file}. "
@@ -175,12 +181,27 @@ def bootstrap_dataset(
175
181
  bins = [bin for _, bin in df.groupby("bin")]
176
182
 
177
183
  # Create per-bin masks once and keep them for all bootstrap samples
178
- zero_mask_bins = []
184
+ zero_mask_bins: list[numpy.ndarray] = []
179
185
  if random_weights:
180
186
  if abs(fraction_zero) > 1e-6:
181
- # Create a random mask for reflections with zero weight
187
+ # Create per-bin zero masks with an exact number of zero weights.
188
+ # This keeps the zero fraction deterministic for each bin while still
189
+ # being reproducible for a given seed.
182
190
  rng = numpy.random.default_rng(seeds[0])
183
- zero_mask_bins = [rng.random(len(bin)) < fraction_zero for bin in bins]
191
+ zero_mask_bins = []
192
+ for bin in bins:
193
+ n_bin = len(bin)
194
+ n_zero = int(round(fraction_zero * n_bin))
195
+ if n_zero <= 0:
196
+ zero_mask_bins.append(numpy.zeros(n_bin, dtype=bool))
197
+ continue
198
+ if n_zero >= n_bin:
199
+ zero_mask_bins.append(numpy.ones(n_bin, dtype=bool))
200
+ continue
201
+ zero_idx = rng.choice(n_bin, size=n_zero, replace=False)
202
+ zero_mask = numpy.zeros(n_bin, dtype=bool)
203
+ zero_mask[zero_idx] = True
204
+ zero_mask_bins.append(zero_mask)
184
205
  elif col_free:
185
206
  # Use free reflections to create a mask for refls with zero weight
186
207
  zero_mask_bins = [bin[col_free].eq(0).to_numpy() for bin in bins]
@@ -192,21 +213,21 @@ def bootstrap_dataset(
192
213
 
193
214
  completeness_list = []
194
215
  for i, seed in enumerate(seeds):
216
+ parts: list[pandas.Series] = []
195
217
  if random_weights:
196
218
  assert len(zero_mask_bins) == len(bins)
197
- parts = []
198
219
  for b, bin in enumerate(bins):
199
- w = resample_random(
220
+ w = resample_dirichlet(
200
221
  len(bin),
201
222
  zero_mask_bins[b],
202
- seed=seed,
223
+ dirichlet_alpha=dirichlet_alpha,
224
+ seed=seed + (b + 1) * 1000000,
203
225
  )
204
226
  parts.append(pandas.Series(w.values, index=bin.index, name="llweight"))
205
227
  df_bootstrap1_weight = pandas.concat(parts).sort_index()
206
228
  else:
207
- parts = []
208
- for bin in bins:
209
- w = resample(len(bin), seed, draw_factor)
229
+ for b, bin in enumerate(bins):
230
+ w = resample(len(bin), seed + (b + 1) * 1000000, draw_factor)
210
231
  parts.append(pandas.Series(w.values, index=bin.index, name="llweight"))
211
232
  df_bootstrap1_weight = pandas.concat(parts).sort_index()
212
233
 
@@ -307,4 +328,4 @@ def bootstrap_dataset(
307
328
  completeness_orig = n_unique_orig / n_unique_expected
308
329
  logging.info(f"Completeness of the original dataset: {completeness_orig:.2%}\n")
309
330
 
310
- return mtzs_out
331
+ return mtzs_out, mtz_filtered_name