dspin 1.3.5__tar.gz → 1.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dspin
3
- Version: 1.3.5
3
+ Version: 1.4.2
4
4
  Summary: Regulatory network models from single-cell perturbation profiling
5
5
  Home-page: https://github.com/JialongJiang/DSPIN
6
6
  Author: Jialong Jiang
@@ -18,6 +18,8 @@ The MATLAB code can be downloaded from the folder "DSPIN_matlab", and directly e
18
18
 
19
19
  ## Demos
20
20
 
21
+ A tutorial intended for an invited book chapter is available at [bioRxiv](https://www.biorxiv.org/content/10.1101/2025.07.28.667326).
22
+
21
23
  Two demos of D-SPIN are available on Google Colab.
22
24
 
23
25
  The first demo reconstructs the regulatory network of simulated hematopoietic stem cell (HSC) differentiation network with perturbations using the BEELINE framework (Pratapa, Aditya, et al. Nature methods, 2020).
@@ -170,6 +172,8 @@ model_gene.program_regulator_discovery(
170
172
 
171
173
  1. Jiang, Jialong, et al. "D-SPIN constructs gene regulatory network models from multiplexed scRNA-seq data revealing organizing principles of cellular perturbation response." bioRxiv (2023).
172
174
 
173
- 2. Pratapa, Aditya, et al. "Benchmarking algorithms for gene regulatory network inference from single-cell transcriptomic data." Nature methods 17.2 (2020): 147-154.
175
+ 2. Jiang, Jialong, and Thomson, Matt. "A guide to D-SPIN: constructing regulatory network models from single-cell RNA-seq perturbation data." bioRxiv (2025).
176
+
177
+ 3. Pratapa, Aditya, et al. "Benchmarking algorithms for gene regulatory network inference from single-cell transcriptomic data." Nature methods 17.2 (2020): 147-154.
174
178
 
175
- 3. Cui, Ang, et al. "Dictionary of immune responses to cytokines at single-cell resolution." Nature 625.7994 (2024): 377-384.
179
+ 4. Cui, Ang, et al. "Dictionary of immune responses to cytokines at single-cell resolution." Nature 625.7994 (2024): 377-384.
@@ -0,0 +1,21 @@
1
+
2
+ from . import annotate
3
+ from . import compute
4
+ from . import plot
5
+ from . import dspin
6
+ from importlib.metadata import version
7
+
8
+ # Import main classes for easy access
9
+ from .dspin import DSPIN, GeneDSPIN, ProgramDSPIN, AbstractDSPIN
10
+
11
+ __all__ = [
12
+ 'DSPIN',
13
+ 'GeneDSPIN',
14
+ 'ProgramDSPIN',
15
+ 'AbstractDSPIN',
16
+ 'annotate',
17
+ 'compute',
18
+ 'plot'
19
+ ]
20
+
21
+ __version__ = version("DSPIN")
@@ -232,15 +232,16 @@ def onmf(X: np.array, rank: int, max_iter: int = 500) -> Tuple[np.array, np.arra
232
232
  coef_S = AtX / S.dot(AtX.T).dot(S)
233
233
  S = np.nan_to_num(S * coef_S) #, posinf=1e5)
234
234
 
235
- pbar.update(1)
235
+ # pbar.update(1)
236
236
 
237
237
  # Calculate the reconstruction error every 10 iterations and update the
238
238
  # progress bar
239
- if itr % 10 == 0:
239
+ if itr % 20 == 0:
240
240
  error = np.linalg.norm(X - np.dot(A, S), 'fro')
241
241
  self_product = S.dot(S.T)
242
242
  orthogonal_error = np.linalg.norm(self_product - np.diag(np.diag(self_product)), 'fro') / np.linalg.norm(S, 'fro')
243
243
  pbar.set_postfix({"Reconstruction error": f"{error:.2f}", "Orthogonal error": f"{orthogonal_error:.4f}"})
244
+ pbar.update(20)
244
245
 
245
246
  pbar.close()
246
247
 
@@ -278,6 +279,8 @@ def compute_onmf(seed: int,
278
279
  """
279
280
  # Generate a random seed
280
281
  np.random.seed(seed)
282
+
283
+ tqdm._instances.clear()
281
284
  # Factorized Matrices
282
285
  H, W = onmf(gene_matrix, num_spin, max_iter)
283
286
 
@@ -608,7 +611,7 @@ def para_moments(j_mat: np.ndarray,
608
611
  spins[s] = (tmp % 3) - 1
609
612
  tmp //= 3
610
613
 
611
- # Energy: E = −h·s − ½ sᵀJs
614
+ # Energy: E = −h·s − s'Js / 2
612
615
  energy: float = 0.0
613
616
  for ii in range(num_spin):
614
617
  energy -= h_vec[ii] * spins[ii]
@@ -796,6 +799,28 @@ def samp_moments(j_mat: np.ndarray,
796
799
  return corr_para, mean_para
797
800
 
798
801
 
802
+ def _gradient_one_round(kk: int,
803
+ cur_j: np.ndarray,
804
+ cur_h: np.ndarray,
805
+ raw_data_k,
806
+ method: str,
807
+ train_dat: Dict[str, Any]):
808
+ """Helper that compute gradient for a single round."""
809
+ if method == 'pseudo_likelihood':
810
+ j_grad, h_grad = pseudol_gradient(cur_j, cur_h[:, kk:kk + 1], raw_data_k, directed=train_dat['directed'])
811
+ h_grad = h_grad.flatten()
812
+ else:
813
+ if method == 'maximum_likelihood':
814
+ corr_para, mean_para = para_moments(cur_j, cur_h[:, kk])
815
+ elif method == 'mcmc_maximum_likelihood':
816
+ corr_para, mean_para = samp_moments(
817
+ cur_j, cur_h[:, kk], train_dat['mcmc_samplingsz'], train_dat['mcmc_samplingmix'], train_dat['mcmc_samplegap'])
818
+ j_grad = corr_para - raw_data_k[0]
819
+ h_grad = mean_para - raw_data_k[1].flatten()
820
+
821
+ return kk, j_grad, h_grad
822
+
823
+
799
824
  def compute_gradient(cur_j: np.ndarray,
800
825
  cur_h: np.ndarray,
801
826
  raw_data: List[Tuple[np.ndarray, np.ndarray]],
@@ -828,49 +853,32 @@ def compute_gradient(cur_j: np.ndarray,
828
853
  rec_jgrad = np.zeros((num_spin, num_spin, num_round))
829
854
  rec_hgrad = np.zeros((num_spin, num_round))
830
855
 
831
- for kk in range(num_round):
832
- # Computing gradients using the specified method.
833
- if method == 'pseudo_likelihood':
856
+ if method == 'pseudo_likelihood':
857
+ for kk in range(num_round):
858
+ # Computing gradients using the specified method.
834
859
  j_grad, h_grad = pseudol_gradient(
835
860
  cur_j, cur_h[:, kk: kk + 1], raw_data[kk], directed=train_dat['directed'])
836
861
  h_grad = h_grad.flatten()
837
- else:
838
- # Distinguishing between other methods and computing gradients
839
- # accordingly
840
- if method == 'maximum_likelihood':
841
- corr_para, mean_para = para_moments(cur_j, cur_h[:, kk])
842
- elif method == 'mcmc_maximum_likelihood':
843
- corr_para, mean_para = samp_moments(
844
- cur_j, cur_h[:, kk], train_dat['mcmc_samplingsz'], train_dat['mcmc_samplingmix'], train_dat['mcmc_samplegap'])
845
- j_grad = corr_para - raw_data[kk][0]
846
- h_grad = mean_para - raw_data[kk][1].flatten()
847
-
848
- rec_jgrad[:, :, kk] = j_grad
849
- rec_hgrad[:, kk] = h_grad
850
-
851
- return rec_jgrad, rec_hgrad
852
862
 
853
-
854
- def _gradient_one_round(kk: int,
855
- cur_j: np.ndarray,
856
- cur_h: np.ndarray,
857
- raw_data_k,
858
- method: str,
859
- train_dat: Dict[str, Any]):
860
- """Helper that compute gradient for a single round."""
861
- if method == 'pseudo_likelihood':
862
- j_grad, h_grad = pseudol_gradient(cur_j, cur_h[:, kk:kk + 1], raw_data_k, directed=train_dat['directed'])
863
- h_grad = h_grad.flatten()
863
+ rec_jgrad[:, :, kk] = j_grad
864
+ rec_hgrad[:, kk] = h_grad
864
865
  else:
865
- if method == 'maximum_likelihood':
866
- corr_para, mean_para = para_moments(cur_j, cur_h[:, kk])
867
- elif method == 'mcmc_maximum_likelihood':
868
- corr_para, mean_para = samp_moments(
869
- cur_j, cur_h[:, kk], train_dat['mcmc_samplingsz'], train_dat['mcmc_samplingmix'], train_dat['mcmc_samplegap'])
870
- j_grad = corr_para - raw_data_k[0]
871
- h_grad = mean_para - raw_data_k[1].flatten()
872
-
873
- return kk, j_grad, h_grad
866
+ n_jobs = train_dat.get('max_workers', min(num_round, os.cpu_count()))
867
+ chunk = train_dat.get('chunk_size', 1) # rounds per task
868
+ backend = 'loky' # process‑based
869
+
870
+ # print(f'Processing with {n_jobs} workers...')
871
+ with parallel_backend(backend, n_jobs=n_jobs):
872
+ results = Parallel(batch_size=chunk)(
873
+ delayed(_gradient_one_round)(kk, cur_j, cur_h, raw_data[kk], method, train_dat)
874
+ for kk in range(num_round)
875
+ )
876
+
877
+ for kk, j_grad, h_grad in results:
878
+ rec_jgrad[:, :, kk] = j_grad
879
+ rec_hgrad[:, kk] = h_grad
880
+
881
+ return rec_jgrad, rec_hgrad
874
882
 
875
883
 
876
884
  def compute_gradient_para(cur_j: np.ndarray,
@@ -990,10 +998,13 @@ def apply_regularization(rec_jgrad: np.ndarray,
990
998
 
991
999
  if 'lambda_l2_j_prior_mask' in train_dat:
992
1000
  rec_jgrad += train_dat['lambda_l2_j_prior_mask'] * (cur_j * (train_dat['j_prior_mask'] == 0))[:, :, np.newaxis]
1001
+
1002
+ if 'lambda_l1_j_prior_mask' in train_dat:
1003
+ rec_jgrad += train_dat['lambda_l1_j_prior_mask'] * ((cur_j * (train_dat['j_prior_mask'] == 0))[:, :, np.newaxis] / 1e-3).clip(-1, 1)
993
1004
 
994
1005
  if print_regu:
995
1006
  print('Regularization parameters')
996
- all_regu_list = ['lambda_l1_j', 'lambda_l1_h', 'lambda_l2_j', 'lambda_l2_h', 'lambda_l2_j_prior', 'lambda_l2_j_prior_mask', 'lambda_l2_h_rela_prior', 'lambda_l1_h_rela', 'lambda_l2_h_rela']
1007
+ all_regu_list = ['lambda_l1_j', 'lambda_l1_h', 'lambda_l2_j', 'lambda_l2_h', 'lambda_l2_j_prior', 'lambda_l2_j_prior_mask', 'lambda_l2_h_rela_prior', 'lambda_l1_h_rela', 'lambda_l2_h_rela', 'lambda_l1_j_prior_mask']
997
1008
  all_values = [train_dat.get(key, 0) for key in all_regu_list]
998
1009
  for key, value in zip(all_regu_list, all_values):
999
1010
  if value > 0:
@@ -1153,7 +1164,6 @@ def learn_network_adam(raw_data: Any,
1153
1164
  if (counter > backtrack_gap) and (rec_jgrad_sum_norm[counter - 1] > 1.5 * rec_jgrad_sum_norm[counter - 1 - backtrack_gap]):
1154
1165
  if_backtrack = True
1155
1166
 
1156
-
1157
1167
  if (counter > 2 * backtrack_gap) and (counter % backtrack_gap == 0):
1158
1168
 
1159
1169
  est_cur_grad = rec_jgrad_sum_norm[counter - 1 - backtrack_gap: counter - 1].mean()
@@ -1162,7 +1172,6 @@ def learn_network_adam(raw_data: Any,
1162
1172
  if est_cur_grad > 1.05 * est_prev_grad:
1163
1173
  if_backtrack = True
1164
1174
 
1165
-
1166
1175
  if if_backtrack:
1167
1176
  print('Backtracking at epoch %d' % counter)
1168
1177
  backtrack_counter += 1
@@ -1180,8 +1189,7 @@ def learn_network_adam(raw_data: Any,
1180
1189
 
1181
1190
  pbar.close()
1182
1191
 
1183
- # Retrieve parameters corresponding to the minimum gradient norm found
1184
- # during training
1192
+ # Retrieve parameters corresponding to the minimum gradient norm found during training
1185
1193
  trace_epoch = 50 + num_epoch - counter
1186
1194
  pos = num_epoch - trace_epoch + np.argmin(rec_jgrad_sum_norm[- trace_epoch: ])
1187
1195
  # print(pos)
@@ -1221,16 +1229,14 @@ def learn_program_regulators(gene_states: List[np.ndarray],
1221
1229
  num_round = len(gene_states)
1222
1230
 
1223
1231
  num_epoch, stepsz, rec_gap = (
1224
- train_dat.get(
1225
- key, None) for key in [
1232
+ train_dat.get(key, None) for key in [
1226
1233
  "num_epoch", "stepsz", "rec_gap"])
1227
1234
  list_step = np.arange(num_epoch, 0, - rec_gap)[::-1]
1228
1235
 
1229
1236
  save_path = train_dat.get('save_path', None)
1230
1237
 
1231
1238
  backtrack_gap, backtrack_tol = (
1232
- train_dat.get(
1233
- key, None) for key in [
1239
+ train_dat.get(key, None) for key in [
1234
1240
  "backtrack_gap", "backtrack_tol"])
1235
1241
 
1236
1242
  lambda_l1_j, lambda_l2_j = (train_dat.get(key, 0) for key in ["lambda_l1_j", "lambda_l2_j"])
@@ -1246,6 +1252,10 @@ def learn_program_regulators(gene_states: List[np.ndarray],
1246
1252
  samp_weight = samp_weight / np.sum(samp_weight)
1247
1253
 
1248
1254
  # Preallocate gradient norm recording (optional)
1255
+ rec_interaction_all = np.zeros((num_epoch, num_gene, num_program))
1256
+ rec_selfj_all = np.zeros((num_epoch, num_program))
1257
+ rec_selfh_all = np.zeros((num_epoch, num_program))
1258
+
1249
1259
  rec_grad = np.zeros(num_epoch)
1250
1260
 
1251
1261
  m_interaction, v_interaction = np.zeros_like(cur_interaction), np.zeros_like(cur_interaction)
@@ -1265,9 +1275,14 @@ def learn_program_regulators(gene_states: List[np.ndarray],
1265
1275
  rec_interaction_grad = np.zeros((num_round, num_gene, num_program))
1266
1276
  rec_selfj_grad = np.zeros((num_round, num_program))
1267
1277
  rec_selfh_grad = np.zeros((num_round, num_program))
1268
-
1278
+
1279
+ log_adam_grad = {name: deque(maxlen=backtrack_gap)
1280
+ for name in ["m_interaction", "v_interaction", "m_selfj", "v_selfj", "m_selfh", "v_selfh"]}
1281
+
1282
+ tqdm._instances.clear()
1269
1283
  pbar = tqdm(total=num_epoch)
1270
1284
  counter = 1
1285
+ backtrack_counter = 0
1271
1286
 
1272
1287
  while counter <= num_epoch:
1273
1288
 
@@ -1320,15 +1335,52 @@ def learn_program_regulators(gene_states: List[np.ndarray],
1320
1335
  update_val, m_selfh, v_selfh = update_adam(selfh_grad, m_selfh, v_selfh, counter, stepsz)
1321
1336
  cur_selfh = cur_selfh - update_val
1322
1337
 
1338
+ rec_interaction_all[counter - 1, :, :] = cur_interaction
1339
+ rec_selfj_all[counter - 1, :] = cur_selfj
1340
+ rec_selfh_all[counter - 1, :] = cur_selfh
1341
+
1342
+ for name, value in zip(["m_interaction", "v_interaction", "m_selfj", "v_selfj", "m_selfh", "v_selfh"], [m_interaction, v_interaction, m_selfj, v_selfj, m_selfh, v_selfh]):
1343
+ log_adam_grad[name].append(value)
1344
+
1323
1345
  # Record gradient norm (optional)
1324
1346
  rec_grad[counter - 1] = np.linalg.norm(interaction_grad)
1325
1347
 
1326
1348
  if counter in list_step:
1327
1349
  pbar.update(rec_gap)
1328
- pbar.set_postfix({"Network Gradient": f"{rec_grad[counter - 1]:.4f}"})
1350
+ pbar.set_postfix({"Network Gradient": f"{rec_grad[counter - 1]:.6f}"})
1351
+
1352
+ if_backtrack = False
1353
+ if (counter > backtrack_gap) and (rec_grad[counter - 1] > 1.5 * rec_grad[counter - 1 - backtrack_gap]):
1354
+ if_backtrack = True
1355
+
1356
+ if (counter > 2 * backtrack_gap) and (counter % backtrack_gap == 0):
1357
+ est_cur_grad = rec_grad[counter - 1 - backtrack_gap: counter - 1].mean()
1358
+ est_prev_grad = rec_grad[counter - 1 - 2 * backtrack_gap: counter - 1 - backtrack_gap].mean()
1359
+
1360
+ if est_cur_grad > 1.05 * est_prev_grad:
1361
+ if_backtrack = True
1329
1362
 
1330
- counter += 1
1363
+ if if_backtrack:
1364
+ print('Backtracking at epoch %d' % counter)
1365
+ backtrack_counter += 1
1366
+ m_interaction, v_interaction, m_selfj, v_selfj, m_selfh, v_selfh = [log_adam_grad[key][0] for key in ["m_interaction", "v_interaction", "m_selfj", "v_selfj", "m_selfh", "v_selfh"]]
1367
+ counter = counter - backtrack_gap
1368
+ stepsz = stepsz / 4
1369
+ if backtrack_counter > backtrack_tol:
1370
+ print(f'Backtracking more than {backtrack_tol} times, stop training.')
1371
+ break
1372
+ else:
1373
+ counter += 1
1331
1374
 
1375
+ pbar.close()
1376
+
1377
+ # Retrieve parameters corresponding to the minimum gradient norm found during training
1378
+ trace_epoch = 50 + num_epoch - counter
1379
+ pos = num_epoch - trace_epoch + np.argmin(rec_grad[- trace_epoch: ])
1380
+ cur_interaction = rec_interaction_all[pos, :, :]
1381
+ cur_selfj = rec_selfj_all[pos, :]
1382
+ cur_selfh = rec_selfh_all[pos, :]
1383
+
1332
1384
  return cur_interaction, cur_selfj, cur_selfh
1333
1385
 
1334
1386
 
@@ -35,6 +35,9 @@ import warnings
35
35
  import itertools
36
36
  from typing import List, Dict, Optional, Tuple, Union
37
37
  from scipy.stats import skew
38
+ import leidenalg as la
39
+ import igraph as ig
40
+ import matplotlib as mpl
38
41
 
39
42
 
40
43
  class AbstractDSPIN(ABC):
@@ -106,6 +109,14 @@ class AbstractDSPIN(ABC):
106
109
  @property
107
110
  def sample_list(self):
108
111
  return self._samp_list
112
+
113
+ @property
114
+ def name_list(self):
115
+ return self._name_list
116
+
117
+ @property
118
+ def name_list_short(self):
119
+ return self._name_list_short
109
120
 
110
121
  @program_representation_raw.setter
111
122
  def program_representation_raw(self, value):
@@ -131,6 +142,14 @@ class AbstractDSPIN(ABC):
131
142
  def sample_list(self, value):
132
143
  self._samp_list = value
133
144
 
145
+ @name_list.setter
146
+ def name_list(self, value):
147
+ self._name_list = value
148
+
149
+ @name_list_short.setter
150
+ def name_list_short(self, value):
151
+ self._name_list_short = value
152
+
134
153
  def discretize(self,
135
154
  clip_percentile: float = 100,
136
155
  num_init: int = 10) -> None:
@@ -201,13 +220,13 @@ class AbstractDSPIN(ABC):
201
220
  num_spin = self.num_spin
202
221
  raw_data = self._raw_data
203
222
 
204
- if self.example_list is not None:
205
- example_list_ind = [list(self._samp_list).index(samp)
206
- for samp in self.example_list]
207
- self._samp_list = self._samp_list[example_list_ind]
208
- raw_data = raw_data[example_list_ind]
223
+ if self.sample_list_ordered is not None:
224
+ sample_list_ordered_ind = [list(self._samp_list).index(samp)
225
+ for samp in self.sample_list_ordered]
226
+ self._samp_list = self._samp_list[sample_list_ordered_ind]
227
+ raw_data = raw_data[sample_list_ordered_ind]
209
228
  self._raw_data = raw_data
210
- print("Using a subset of {} samples for network inference.".format(len(self._samp_list)))
229
+ print("Using a ordered subset of {} samples for network inference.".format(len(self._samp_list)))
211
230
 
212
231
  num_sample = len(raw_data)
213
232
  params = {'num_epoch': 400,
@@ -241,7 +260,11 @@ class AbstractDSPIN(ABC):
241
260
  method: str = 'auto',
242
261
  directed: bool = False,
243
262
  params: dict = None,
244
- example_list: List[str] = None,
263
+ sample_list_ordered: List[str] = None,
264
+ prior_network: np.ndarray = None,
265
+ perturb_matrix: np.ndarray = None,
266
+ if_control_key: str = 'if_control',
267
+ batch_key: str = 'batch',
245
268
  run_with_matlab: bool = False) -> None:
246
269
  """
247
270
  Perform network inference using a specified method and parameters.
@@ -256,8 +279,16 @@ class AbstractDSPIN(ABC):
256
279
  Whether to infer a directed network. Default is False.
257
280
  params : dict, optional
258
281
  Additional parameters for network inference. Default is None.
259
- example_list : List[str], optional
260
- List of example sample identifiers. Default is None.
282
+ sample_list_ordered : List[str], optional
283
+ List of ordered sample identifiers. Default is None.
284
+ prior_network : np.ndarray, optional
285
+ Binary matrix of preferred network connectivity from prior knowledge. Default is None.
286
+ perturb_matrix : np.ndarray, optional
287
+ Matrix of putative relative responses to control samples. Default is None.
288
+ if_control_key : str, optional
289
+ Column name in `adata.obs` indicating whether a sample is a control. Default is 'if_control'.
290
+ batch_key : str, optional
291
+ Column name in `adata.obs` specifying batch assignment for each sample. Default is 'batch'.
261
292
  run_with_matlab : bool, optional
262
293
  If True, prepares data for MATLAB execution instead of running inference in Python. Default is False.
263
294
  """
@@ -275,8 +306,8 @@ class AbstractDSPIN(ABC):
275
306
  if method == 'auto':
276
307
  samp_list = np.unique(self.adata.obs[sample_id_key])
277
308
  num_sample = len(samp_list)
278
- if example_list is not None:
279
- num_sample = len(example_list)
309
+ if sample_list_ordered is not None:
310
+ num_sample = len(sample_list_ordered)
280
311
  if num_sample > 30:
281
312
  method = 'pseudo_likelihood'
282
313
  else:
@@ -296,9 +327,32 @@ class AbstractDSPIN(ABC):
296
327
  else:
297
328
  self.raw_data_corr(sample_id_key)
298
329
 
299
- self.example_list = example_list
330
+ self.sample_list_ordered = sample_list_ordered
300
331
 
301
332
  train_dat = self.default_params(method)
333
+
334
+ if prior_network is not None:
335
+ train_dat['j_prior_mask'] = prior_network
336
+ train_dat['lambda_l1_j_prior_mask'] = 0.01
337
+
338
+ if perturb_matrix is not None:
339
+ if if_control_key not in self.adata.obs.columns:
340
+ raise ValueError(f"Column '{if_control_key}' not found in adata.obs.")
341
+ if batch_key not in self.adata.obs.columns:
342
+ raise ValueError(f"Column '{batch_key}' not found in adata.obs.")
343
+
344
+ sample_list = self._samp_list
345
+ sample_to_control = self.adata.obs.groupby(sample_id_key, observed=False)[if_control_key].first().to_dict()
346
+ sample_to_batch = self.adata.obs.groupby(sample_id_key, observed=False)[batch_key].first().to_dict()
347
+
348
+ if_control = np.array([sample_to_control[sample] for sample in sample_list])
349
+ batch_index = np.array([sample_to_batch[sample] for sample in sample_list])
350
+
351
+ train_dat['perturb_matrix'] = perturb_matrix
352
+ train_dat['if_control'] = if_control
353
+ train_dat['batch_index'] = batch_index
354
+ train_dat['lambda_l2_h_rela_prior'] = 0.5
355
+
302
356
  train_dat['directed'] = directed
303
357
  if params is not None:
304
358
  train_dat.update(params)
@@ -390,9 +444,10 @@ class GeneDSPIN(AbstractDSPIN):
390
444
  cur_gene = self._onmf_rep_ori[:, ii]
391
445
  gene_skew[ii] = skew(cur_gene, bias=False)
392
446
 
393
- if gene_skew.mean() > 2:
447
+ print(f'Skewness of the gene expression: {gene_skew.mean().round(3)}')
448
+
449
+ if gene_skew.mean() >= 1.5:
394
450
  default_clip = 95
395
- print("Input gene expression seem to be heavy-tailed. Using 95 percentile for clipping before discretization.")
396
451
  else:
397
452
  default_clip = 100
398
453
 
@@ -400,13 +455,20 @@ class GeneDSPIN(AbstractDSPIN):
400
455
  'clip_percentile': default_clip,
401
456
  'num_init': 10}
402
457
  dparams.update(discretize_params)
403
-
458
+
459
+ self.name_list = self.adata.var_names.tolist()
460
+ self.name_list_short = self.adata.var_names.tolist()
461
+
404
462
  if dparams['use_default_discretize']:
463
+
464
+ if dparams['clip_percentile'] < 100:
465
+ print(f"Input gene expression seem to be heavy-tailed. Using {dparams['clip_percentile']} percentile for clipping before discretization.")
466
+
405
467
  self.discretize(clip_percentile=dparams['clip_percentile'],
406
468
  num_init=dparams['num_init'])
407
469
 
408
470
  def program_regulator_discovery(self,
409
- program_representation_raw: np.ndarray,
471
+ program_representation: np.ndarray,
410
472
  sample_id_key: str = 'sample_id',
411
473
  params: dict = None):
412
474
  """
@@ -414,7 +476,7 @@ class GeneDSPIN(AbstractDSPIN):
414
476
 
415
477
  Parameters
416
478
  ----------
417
- program_representation_raw : np.ndarray
479
+ program_representation : np.ndarray
418
480
  The gene program representation.
419
481
  sample_id_key : str, optional
420
482
  Column name in `adata.obs` specifying sample identifiers. Default is 'sample_id'.
@@ -422,7 +484,7 @@ class GeneDSPIN(AbstractDSPIN):
422
484
  Additional parameters for regression. Default is None.
423
485
  """
424
486
 
425
- program_states, samp_list = sample_states(self.adata.obs[sample_id_key], program_representation_raw)
487
+ program_states, samp_list = sample_states(self.adata.obs[sample_id_key], program_representation)
426
488
 
427
489
  gene_states, _ = sample_states(self.adata.obs[sample_id_key], self._onmf_rep_tri)
428
490
 
@@ -430,7 +492,9 @@ class GeneDSPIN(AbstractDSPIN):
430
492
  'num_epoch': 500,
431
493
  'stepsz': 0.01,
432
494
  'lambda_l1_interaction': 0.01,
433
- 'rec_gap': 10
495
+ 'rec_gap': 10,
496
+ 'backtrack_gap': 40,
497
+ 'backtrack_tol': 5
434
498
  }
435
499
 
436
500
  if params is not None:
@@ -664,11 +728,11 @@ class ProgramDSPIN(AbstractDSPIN):
664
728
  self.prior_programs_ind = prior_program_ind
665
729
 
666
730
  num_onmf_components = onmf_parameters['num_onmf_components']
731
+
732
+ os.makedirs(self.save_path + 'onmf/', exist_ok=True)
667
733
 
668
734
  if mode == 'compute_summary' or mode == 'compute_only':
669
735
 
670
- os.makedirs(self.save_path + 'onmf/', exist_ok=True)
671
-
672
736
  cur_gene_matrix = adata.X[:, prior_program_mask]
673
737
  cluster_label_raw = adata.obs[cluster_key].values
674
738
  cluster_label = pd.factorize(cluster_label_raw)[0]
@@ -709,10 +773,11 @@ class ProgramDSPIN(AbstractDSPIN):
709
773
  # Save the gene programs to a CSV file
710
774
  file_path = onmf_to_csv(onmf_summary.components_, adata.var_names, self.save_path + f'gene_programs_total_{num_spin}_onmf_{num_onmf_components}_prior_{len(prior_program_ind)}.csv', thres=0.01)
711
775
  print('Gene program information saved to {}'.format(file_path))
712
- file_wpath = onmf_to_csv(onmf_summary.components_, adata.var_names, self.save_path + f'gene_programs_total_{num_spin}_onmf_{num_onmf_components}_prior_{len(prior_program_ind)}_weights.csv', thres=0.01, if_write_weights=True)
713
- # print('Gene programs saved to {}'.format(file_path))
714
776
  self.gene_program_csv = file_path
715
777
 
778
+ file_wpath = onmf_to_csv(onmf_summary.components_, adata.var_names, self.save_path + f'gene_programs_total_{num_spin}_onmf_{num_onmf_components}_prior_{len(prior_program_ind)}_weights.csv', thres=0.01, if_write_weights=True)
779
+ # print('Gene programs saved to {}'.format(file_path))
780
+
716
781
  gene_matrix = self.adata.X
717
782
  if issparse(gene_matrix):
718
783
  gene_matrix = np.asarray(gene_matrix.toarray()).astype(np.float64)
@@ -727,6 +792,20 @@ class ProgramDSPIN(AbstractDSPIN):
727
792
  self.discretize(clip_percentile=dparams['clip_percentile'],
728
793
  num_init=dparams['num_init'])
729
794
 
795
+ num_display_gene = 5
796
+ gene_pd = pd.read_csv(self.gene_program_csv, index_col=None)
797
+
798
+ top_gene_list = [gene_pd.iloc[:, ii].tolist()[:num_display_gene] for ii in range(num_spin)]
799
+ # remove nan
800
+ top_gene_list_filtered = []
801
+ for sub_list in top_gene_list:
802
+ sub_list_filtered = [gene for gene in sub_list if str(gene) != 'nan']
803
+ top_gene_list_filtered.append(sub_list_filtered)
804
+
805
+ program_names = [f'P{ii}-' + ','.join(top_gene_list_filtered[ii]) for ii in range(num_spin)]
806
+ self.name_list = program_names
807
+ self.name_list_short = [name.split('-')[0] for name in program_names]
808
+
730
809
 
731
810
  class DSPIN(object):
732
811
  """