dspin 1.3.5__tar.gz → 1.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dspin-1.3.5/dspin.egg-info → dspin-1.4.2}/PKG-INFO +1 -1
- {dspin-1.3.5 → dspin-1.4.2}/README.md +6 -2
- dspin-1.4.2/dspin/__init__.py +21 -0
- {dspin-1.3.5 → dspin-1.4.2}/dspin/compute.py +105 -53
- {dspin-1.3.5 → dspin-1.4.2}/dspin/dspin.py +102 -23
- dspin-1.4.2/dspin/plot.py +776 -0
- {dspin-1.3.5 → dspin-1.4.2/dspin.egg-info}/PKG-INFO +1 -1
- {dspin-1.3.5 → dspin-1.4.2}/setup.py +1 -1
- dspin-1.3.5/dspin/__init__.py +0 -5
- dspin-1.3.5/dspin/plot.py +0 -559
- {dspin-1.3.5 → dspin-1.4.2}/LICENSE +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/__pycache__/__init__.cpython-310.pyc +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/__pycache__/__init__.cpython-37.pyc +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/__pycache__/compute.cpython-310.pyc +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/__pycache__/compute.cpython-37.pyc +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/__pycache__/dspin_abstract.cpython-310.pyc +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/__pycache__/plot.cpython-310.pyc +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/__pycache__/plotting.cpython-37.pyc +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/dspin/annotate.py +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/dspin.egg-info/SOURCES.txt +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/dspin.egg-info/dependency_links.txt +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/dspin.egg-info/requires.txt +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/dspin.egg-info/top_level.txt +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/requirements.txt +0 -0
- {dspin-1.3.5 → dspin-1.4.2}/setup.cfg +0 -0
|
@@ -18,6 +18,8 @@ The MATLAB code can be downloaded from the folder "DSPIN_matlab", and directly e
|
|
|
18
18
|
|
|
19
19
|
## Demos
|
|
20
20
|
|
|
21
|
+
A tutorial intended for an invited book chapter is available at [bioRxiv](https://www.biorxiv.org/content/10.1101/2025.07.28.667326).
|
|
22
|
+
|
|
21
23
|
Two demos of D-SPIN are available on Google Colab.
|
|
22
24
|
|
|
23
25
|
The first demo reconstructs the regulatory network of simulated hematopoietic stem cell (HSC) differentiation network with perturbations using the BEELINE framework (Pratapa, Aditya, et al. Nature methods, 2020).
|
|
@@ -170,6 +172,8 @@ model_gene.program_regulator_discovery(
|
|
|
170
172
|
|
|
171
173
|
1. Jiang, Jialong, et al. "D-SPIN constructs gene regulatory network models from multiplexed scRNA-seq data revealing organizing principles of cellular perturbation response." bioRxiv (2023).
|
|
172
174
|
|
|
173
|
-
2.
|
|
175
|
+
2. Jiang, Jialong, and Thomson, Matt. "A guide to D-SPIN: constructing regulatory network models from single-cell RNA-seq perturbation data." bioRxiv (2025).
|
|
176
|
+
|
|
177
|
+
3. Pratapa, Aditya, et al. "Benchmarking algorithms for gene regulatory network inference from single-cell transcriptomic data." Nature methods 17.2 (2020): 147-154.
|
|
174
178
|
|
|
175
|
-
|
|
179
|
+
4. Cui, Ang, et al. "Dictionary of immune responses to cytokines at single-cell resolution." Nature 625.7994 (2024): 377-384.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
|
|
2
|
+
from . import annotate
|
|
3
|
+
from . import compute
|
|
4
|
+
from . import plot
|
|
5
|
+
from . import dspin
|
|
6
|
+
from importlib.metadata import version
|
|
7
|
+
|
|
8
|
+
# Import main classes for easy access
|
|
9
|
+
from .dspin import DSPIN, GeneDSPIN, ProgramDSPIN, AbstractDSPIN
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
'DSPIN',
|
|
13
|
+
'GeneDSPIN',
|
|
14
|
+
'ProgramDSPIN',
|
|
15
|
+
'AbstractDSPIN',
|
|
16
|
+
'annotate',
|
|
17
|
+
'compute',
|
|
18
|
+
'plot'
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
__version__ = version("DSPIN")
|
|
@@ -232,15 +232,16 @@ def onmf(X: np.array, rank: int, max_iter: int = 500) -> Tuple[np.array, np.arra
|
|
|
232
232
|
coef_S = AtX / S.dot(AtX.T).dot(S)
|
|
233
233
|
S = np.nan_to_num(S * coef_S) #, posinf=1e5)
|
|
234
234
|
|
|
235
|
-
pbar.update(1)
|
|
235
|
+
# pbar.update(1)
|
|
236
236
|
|
|
237
237
|
# Calculate the reconstruction error every 10 iterations and update the
|
|
238
238
|
# progress bar
|
|
239
|
-
if itr %
|
|
239
|
+
if itr % 20 == 0:
|
|
240
240
|
error = np.linalg.norm(X - np.dot(A, S), 'fro')
|
|
241
241
|
self_product = S.dot(S.T)
|
|
242
242
|
orthogonal_error = np.linalg.norm(self_product - np.diag(np.diag(self_product)), 'fro') / np.linalg.norm(S, 'fro')
|
|
243
243
|
pbar.set_postfix({"Reconstruction error": f"{error:.2f}", "Orthogonal error": f"{orthogonal_error:.4f}"})
|
|
244
|
+
pbar.update(20)
|
|
244
245
|
|
|
245
246
|
pbar.close()
|
|
246
247
|
|
|
@@ -278,6 +279,8 @@ def compute_onmf(seed: int,
|
|
|
278
279
|
"""
|
|
279
280
|
# Generate a random seed
|
|
280
281
|
np.random.seed(seed)
|
|
282
|
+
|
|
283
|
+
tqdm._instances.clear()
|
|
281
284
|
# Factorized Matrices
|
|
282
285
|
H, W = onmf(gene_matrix, num_spin, max_iter)
|
|
283
286
|
|
|
@@ -608,7 +611,7 @@ def para_moments(j_mat: np.ndarray,
|
|
|
608
611
|
spins[s] = (tmp % 3) - 1
|
|
609
612
|
tmp //= 3
|
|
610
613
|
|
|
611
|
-
# Energy: E = −h·s −
|
|
614
|
+
# Energy: E = −h·s − s'Js / 2
|
|
612
615
|
energy: float = 0.0
|
|
613
616
|
for ii in range(num_spin):
|
|
614
617
|
energy -= h_vec[ii] * spins[ii]
|
|
@@ -796,6 +799,28 @@ def samp_moments(j_mat: np.ndarray,
|
|
|
796
799
|
return corr_para, mean_para
|
|
797
800
|
|
|
798
801
|
|
|
802
|
+
def _gradient_one_round(kk: int,
|
|
803
|
+
cur_j: np.ndarray,
|
|
804
|
+
cur_h: np.ndarray,
|
|
805
|
+
raw_data_k,
|
|
806
|
+
method: str,
|
|
807
|
+
train_dat: Dict[str, Any]):
|
|
808
|
+
"""Helper that compute gradient for a single round."""
|
|
809
|
+
if method == 'pseudo_likelihood':
|
|
810
|
+
j_grad, h_grad = pseudol_gradient(cur_j, cur_h[:, kk:kk + 1], raw_data_k, directed=train_dat['directed'])
|
|
811
|
+
h_grad = h_grad.flatten()
|
|
812
|
+
else:
|
|
813
|
+
if method == 'maximum_likelihood':
|
|
814
|
+
corr_para, mean_para = para_moments(cur_j, cur_h[:, kk])
|
|
815
|
+
elif method == 'mcmc_maximum_likelihood':
|
|
816
|
+
corr_para, mean_para = samp_moments(
|
|
817
|
+
cur_j, cur_h[:, kk], train_dat['mcmc_samplingsz'], train_dat['mcmc_samplingmix'], train_dat['mcmc_samplegap'])
|
|
818
|
+
j_grad = corr_para - raw_data_k[0]
|
|
819
|
+
h_grad = mean_para - raw_data_k[1].flatten()
|
|
820
|
+
|
|
821
|
+
return kk, j_grad, h_grad
|
|
822
|
+
|
|
823
|
+
|
|
799
824
|
def compute_gradient(cur_j: np.ndarray,
|
|
800
825
|
cur_h: np.ndarray,
|
|
801
826
|
raw_data: List[Tuple[np.ndarray, np.ndarray]],
|
|
@@ -828,49 +853,32 @@ def compute_gradient(cur_j: np.ndarray,
|
|
|
828
853
|
rec_jgrad = np.zeros((num_spin, num_spin, num_round))
|
|
829
854
|
rec_hgrad = np.zeros((num_spin, num_round))
|
|
830
855
|
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
856
|
+
if method == 'pseudo_likelihood':
|
|
857
|
+
for kk in range(num_round):
|
|
858
|
+
# Computing gradients using the specified method.
|
|
834
859
|
j_grad, h_grad = pseudol_gradient(
|
|
835
860
|
cur_j, cur_h[:, kk: kk + 1], raw_data[kk], directed=train_dat['directed'])
|
|
836
861
|
h_grad = h_grad.flatten()
|
|
837
|
-
else:
|
|
838
|
-
# Distinguishing between other methods and computing gradients
|
|
839
|
-
# accordingly
|
|
840
|
-
if method == 'maximum_likelihood':
|
|
841
|
-
corr_para, mean_para = para_moments(cur_j, cur_h[:, kk])
|
|
842
|
-
elif method == 'mcmc_maximum_likelihood':
|
|
843
|
-
corr_para, mean_para = samp_moments(
|
|
844
|
-
cur_j, cur_h[:, kk], train_dat['mcmc_samplingsz'], train_dat['mcmc_samplingmix'], train_dat['mcmc_samplegap'])
|
|
845
|
-
j_grad = corr_para - raw_data[kk][0]
|
|
846
|
-
h_grad = mean_para - raw_data[kk][1].flatten()
|
|
847
|
-
|
|
848
|
-
rec_jgrad[:, :, kk] = j_grad
|
|
849
|
-
rec_hgrad[:, kk] = h_grad
|
|
850
|
-
|
|
851
|
-
return rec_jgrad, rec_hgrad
|
|
852
862
|
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
cur_j: np.ndarray,
|
|
856
|
-
cur_h: np.ndarray,
|
|
857
|
-
raw_data_k,
|
|
858
|
-
method: str,
|
|
859
|
-
train_dat: Dict[str, Any]):
|
|
860
|
-
"""Helper that compute gradient for a single round."""
|
|
861
|
-
if method == 'pseudo_likelihood':
|
|
862
|
-
j_grad, h_grad = pseudol_gradient(cur_j, cur_h[:, kk:kk + 1], raw_data_k, directed=train_dat['directed'])
|
|
863
|
-
h_grad = h_grad.flatten()
|
|
863
|
+
rec_jgrad[:, :, kk] = j_grad
|
|
864
|
+
rec_hgrad[:, kk] = h_grad
|
|
864
865
|
else:
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
866
|
+
n_jobs = train_dat.get('max_workers', min(num_round, os.cpu_count()))
|
|
867
|
+
chunk = train_dat.get('chunk_size', 1) # rounds per task
|
|
868
|
+
backend = 'loky' # process‑based
|
|
869
|
+
|
|
870
|
+
# print(f'Processing with {n_jobs} workers...')
|
|
871
|
+
with parallel_backend(backend, n_jobs=n_jobs):
|
|
872
|
+
results = Parallel(batch_size=chunk)(
|
|
873
|
+
delayed(_gradient_one_round)(kk, cur_j, cur_h, raw_data[kk], method, train_dat)
|
|
874
|
+
for kk in range(num_round)
|
|
875
|
+
)
|
|
876
|
+
|
|
877
|
+
for kk, j_grad, h_grad in results:
|
|
878
|
+
rec_jgrad[:, :, kk] = j_grad
|
|
879
|
+
rec_hgrad[:, kk] = h_grad
|
|
880
|
+
|
|
881
|
+
return rec_jgrad, rec_hgrad
|
|
874
882
|
|
|
875
883
|
|
|
876
884
|
def compute_gradient_para(cur_j: np.ndarray,
|
|
@@ -990,10 +998,13 @@ def apply_regularization(rec_jgrad: np.ndarray,
|
|
|
990
998
|
|
|
991
999
|
if 'lambda_l2_j_prior_mask' in train_dat:
|
|
992
1000
|
rec_jgrad += train_dat['lambda_l2_j_prior_mask'] * (cur_j * (train_dat['j_prior_mask'] == 0))[:, :, np.newaxis]
|
|
1001
|
+
|
|
1002
|
+
if 'lambda_l1_j_prior_mask' in train_dat:
|
|
1003
|
+
rec_jgrad += train_dat['lambda_l1_j_prior_mask'] * ((cur_j * (train_dat['j_prior_mask'] == 0))[:, :, np.newaxis] / 1e-3).clip(-1, 1)
|
|
993
1004
|
|
|
994
1005
|
if print_regu:
|
|
995
1006
|
print('Regularization parameters')
|
|
996
|
-
all_regu_list = ['lambda_l1_j', 'lambda_l1_h', 'lambda_l2_j', 'lambda_l2_h', 'lambda_l2_j_prior', 'lambda_l2_j_prior_mask', 'lambda_l2_h_rela_prior', 'lambda_l1_h_rela', 'lambda_l2_h_rela']
|
|
1007
|
+
all_regu_list = ['lambda_l1_j', 'lambda_l1_h', 'lambda_l2_j', 'lambda_l2_h', 'lambda_l2_j_prior', 'lambda_l2_j_prior_mask', 'lambda_l2_h_rela_prior', 'lambda_l1_h_rela', 'lambda_l2_h_rela', 'lambda_l1_j_prior_mask']
|
|
997
1008
|
all_values = [train_dat.get(key, 0) for key in all_regu_list]
|
|
998
1009
|
for key, value in zip(all_regu_list, all_values):
|
|
999
1010
|
if value > 0:
|
|
@@ -1153,7 +1164,6 @@ def learn_network_adam(raw_data: Any,
|
|
|
1153
1164
|
if (counter > backtrack_gap) and (rec_jgrad_sum_norm[counter - 1] > 1.5 * rec_jgrad_sum_norm[counter - 1 - backtrack_gap]):
|
|
1154
1165
|
if_backtrack = True
|
|
1155
1166
|
|
|
1156
|
-
|
|
1157
1167
|
if (counter > 2 * backtrack_gap) and (counter % backtrack_gap == 0):
|
|
1158
1168
|
|
|
1159
1169
|
est_cur_grad = rec_jgrad_sum_norm[counter - 1 - backtrack_gap: counter - 1].mean()
|
|
@@ -1162,7 +1172,6 @@ def learn_network_adam(raw_data: Any,
|
|
|
1162
1172
|
if est_cur_grad > 1.05 * est_prev_grad:
|
|
1163
1173
|
if_backtrack = True
|
|
1164
1174
|
|
|
1165
|
-
|
|
1166
1175
|
if if_backtrack:
|
|
1167
1176
|
print('Backtracking at epoch %d' % counter)
|
|
1168
1177
|
backtrack_counter += 1
|
|
@@ -1180,8 +1189,7 @@ def learn_network_adam(raw_data: Any,
|
|
|
1180
1189
|
|
|
1181
1190
|
pbar.close()
|
|
1182
1191
|
|
|
1183
|
-
# Retrieve parameters corresponding to the minimum gradient norm found
|
|
1184
|
-
# during training
|
|
1192
|
+
# Retrieve parameters corresponding to the minimum gradient norm found during training
|
|
1185
1193
|
trace_epoch = 50 + num_epoch - counter
|
|
1186
1194
|
pos = num_epoch - trace_epoch + np.argmin(rec_jgrad_sum_norm[- trace_epoch: ])
|
|
1187
1195
|
# print(pos)
|
|
@@ -1221,16 +1229,14 @@ def learn_program_regulators(gene_states: List[np.ndarray],
|
|
|
1221
1229
|
num_round = len(gene_states)
|
|
1222
1230
|
|
|
1223
1231
|
num_epoch, stepsz, rec_gap = (
|
|
1224
|
-
train_dat.get(
|
|
1225
|
-
key, None) for key in [
|
|
1232
|
+
train_dat.get(key, None) for key in [
|
|
1226
1233
|
"num_epoch", "stepsz", "rec_gap"])
|
|
1227
1234
|
list_step = np.arange(num_epoch, 0, - rec_gap)[::-1]
|
|
1228
1235
|
|
|
1229
1236
|
save_path = train_dat.get('save_path', None)
|
|
1230
1237
|
|
|
1231
1238
|
backtrack_gap, backtrack_tol = (
|
|
1232
|
-
train_dat.get(
|
|
1233
|
-
key, None) for key in [
|
|
1239
|
+
train_dat.get(key, None) for key in [
|
|
1234
1240
|
"backtrack_gap", "backtrack_tol"])
|
|
1235
1241
|
|
|
1236
1242
|
lambda_l1_j, lambda_l2_j = (train_dat.get(key, 0) for key in ["lambda_l1_j", "lambda_l2_j"])
|
|
@@ -1246,6 +1252,10 @@ def learn_program_regulators(gene_states: List[np.ndarray],
|
|
|
1246
1252
|
samp_weight = samp_weight / np.sum(samp_weight)
|
|
1247
1253
|
|
|
1248
1254
|
# Preallocate gradient norm recording (optional)
|
|
1255
|
+
rec_interaction_all = np.zeros((num_epoch, num_gene, num_program))
|
|
1256
|
+
rec_selfj_all = np.zeros((num_epoch, num_program))
|
|
1257
|
+
rec_selfh_all = np.zeros((num_epoch, num_program))
|
|
1258
|
+
|
|
1249
1259
|
rec_grad = np.zeros(num_epoch)
|
|
1250
1260
|
|
|
1251
1261
|
m_interaction, v_interaction = np.zeros_like(cur_interaction), np.zeros_like(cur_interaction)
|
|
@@ -1265,9 +1275,14 @@ def learn_program_regulators(gene_states: List[np.ndarray],
|
|
|
1265
1275
|
rec_interaction_grad = np.zeros((num_round, num_gene, num_program))
|
|
1266
1276
|
rec_selfj_grad = np.zeros((num_round, num_program))
|
|
1267
1277
|
rec_selfh_grad = np.zeros((num_round, num_program))
|
|
1268
|
-
|
|
1278
|
+
|
|
1279
|
+
log_adam_grad = {name: deque(maxlen=backtrack_gap)
|
|
1280
|
+
for name in ["m_interaction", "v_interaction", "m_selfj", "v_selfj", "m_selfh", "v_selfh"]}
|
|
1281
|
+
|
|
1282
|
+
tqdm._instances.clear()
|
|
1269
1283
|
pbar = tqdm(total=num_epoch)
|
|
1270
1284
|
counter = 1
|
|
1285
|
+
backtrack_counter = 0
|
|
1271
1286
|
|
|
1272
1287
|
while counter <= num_epoch:
|
|
1273
1288
|
|
|
@@ -1320,15 +1335,52 @@ def learn_program_regulators(gene_states: List[np.ndarray],
|
|
|
1320
1335
|
update_val, m_selfh, v_selfh = update_adam(selfh_grad, m_selfh, v_selfh, counter, stepsz)
|
|
1321
1336
|
cur_selfh = cur_selfh - update_val
|
|
1322
1337
|
|
|
1338
|
+
rec_interaction_all[counter - 1, :, :] = cur_interaction
|
|
1339
|
+
rec_selfj_all[counter - 1, :] = cur_selfj
|
|
1340
|
+
rec_selfh_all[counter - 1, :] = cur_selfh
|
|
1341
|
+
|
|
1342
|
+
for name, value in zip(["m_interaction", "v_interaction", "m_selfj", "v_selfj", "m_selfh", "v_selfh"], [m_interaction, v_interaction, m_selfj, v_selfj, m_selfh, v_selfh]):
|
|
1343
|
+
log_adam_grad[name].append(value)
|
|
1344
|
+
|
|
1323
1345
|
# Record gradient norm (optional)
|
|
1324
1346
|
rec_grad[counter - 1] = np.linalg.norm(interaction_grad)
|
|
1325
1347
|
|
|
1326
1348
|
if counter in list_step:
|
|
1327
1349
|
pbar.update(rec_gap)
|
|
1328
|
-
pbar.set_postfix({"Network Gradient": f"{rec_grad[counter - 1]:.
|
|
1350
|
+
pbar.set_postfix({"Network Gradient": f"{rec_grad[counter - 1]:.6f}"})
|
|
1351
|
+
|
|
1352
|
+
if_backtrack = False
|
|
1353
|
+
if (counter > backtrack_gap) and (rec_grad[counter - 1] > 1.5 * rec_grad[counter - 1 - backtrack_gap]):
|
|
1354
|
+
if_backtrack = True
|
|
1355
|
+
|
|
1356
|
+
if (counter > 2 * backtrack_gap) and (counter % backtrack_gap == 0):
|
|
1357
|
+
est_cur_grad = rec_grad[counter - 1 - backtrack_gap: counter - 1].mean()
|
|
1358
|
+
est_prev_grad = rec_grad[counter - 1 - 2 * backtrack_gap: counter - 1 - backtrack_gap].mean()
|
|
1359
|
+
|
|
1360
|
+
if est_cur_grad > 1.05 * est_prev_grad:
|
|
1361
|
+
if_backtrack = True
|
|
1329
1362
|
|
|
1330
|
-
|
|
1363
|
+
if if_backtrack:
|
|
1364
|
+
print('Backtracking at epoch %d' % counter)
|
|
1365
|
+
backtrack_counter += 1
|
|
1366
|
+
m_interaction, v_interaction, m_selfj, v_selfj, m_selfh, v_selfh = [log_adam_grad[key][0] for key in ["m_interaction", "v_interaction", "m_selfj", "v_selfj", "m_selfh", "v_selfh"]]
|
|
1367
|
+
counter = counter - backtrack_gap
|
|
1368
|
+
stepsz = stepsz / 4
|
|
1369
|
+
if backtrack_counter > backtrack_tol:
|
|
1370
|
+
print(f'Backtracking more than {backtrack_tol} times, stop training.')
|
|
1371
|
+
break
|
|
1372
|
+
else:
|
|
1373
|
+
counter += 1
|
|
1331
1374
|
|
|
1375
|
+
pbar.close()
|
|
1376
|
+
|
|
1377
|
+
# Retrieve parameters corresponding to the minimum gradient norm found during training
|
|
1378
|
+
trace_epoch = 50 + num_epoch - counter
|
|
1379
|
+
pos = num_epoch - trace_epoch + np.argmin(rec_grad[- trace_epoch: ])
|
|
1380
|
+
cur_interaction = rec_interaction_all[pos, :, :]
|
|
1381
|
+
cur_selfj = rec_selfj_all[pos, :]
|
|
1382
|
+
cur_selfh = rec_selfh_all[pos, :]
|
|
1383
|
+
|
|
1332
1384
|
return cur_interaction, cur_selfj, cur_selfh
|
|
1333
1385
|
|
|
1334
1386
|
|
|
@@ -35,6 +35,9 @@ import warnings
|
|
|
35
35
|
import itertools
|
|
36
36
|
from typing import List, Dict, Optional, Tuple, Union
|
|
37
37
|
from scipy.stats import skew
|
|
38
|
+
import leidenalg as la
|
|
39
|
+
import igraph as ig
|
|
40
|
+
import matplotlib as mpl
|
|
38
41
|
|
|
39
42
|
|
|
40
43
|
class AbstractDSPIN(ABC):
|
|
@@ -106,6 +109,14 @@ class AbstractDSPIN(ABC):
|
|
|
106
109
|
@property
|
|
107
110
|
def sample_list(self):
|
|
108
111
|
return self._samp_list
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def name_list(self):
|
|
115
|
+
return self._name_list
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def name_list_short(self):
|
|
119
|
+
return self._name_list_short
|
|
109
120
|
|
|
110
121
|
@program_representation_raw.setter
|
|
111
122
|
def program_representation_raw(self, value):
|
|
@@ -131,6 +142,14 @@ class AbstractDSPIN(ABC):
|
|
|
131
142
|
def sample_list(self, value):
|
|
132
143
|
self._samp_list = value
|
|
133
144
|
|
|
145
|
+
@name_list.setter
|
|
146
|
+
def name_list(self, value):
|
|
147
|
+
self._name_list = value
|
|
148
|
+
|
|
149
|
+
@name_list_short.setter
|
|
150
|
+
def name_list_short(self, value):
|
|
151
|
+
self._name_list_short = value
|
|
152
|
+
|
|
134
153
|
def discretize(self,
|
|
135
154
|
clip_percentile: float = 100,
|
|
136
155
|
num_init: int = 10) -> None:
|
|
@@ -201,13 +220,13 @@ class AbstractDSPIN(ABC):
|
|
|
201
220
|
num_spin = self.num_spin
|
|
202
221
|
raw_data = self._raw_data
|
|
203
222
|
|
|
204
|
-
if self.
|
|
205
|
-
|
|
206
|
-
for samp in self.
|
|
207
|
-
self._samp_list = self._samp_list[
|
|
208
|
-
raw_data = raw_data[
|
|
223
|
+
if self.sample_list_ordered is not None:
|
|
224
|
+
sample_list_ordered_ind = [list(self._samp_list).index(samp)
|
|
225
|
+
for samp in self.sample_list_ordered]
|
|
226
|
+
self._samp_list = self._samp_list[sample_list_ordered_ind]
|
|
227
|
+
raw_data = raw_data[sample_list_ordered_ind]
|
|
209
228
|
self._raw_data = raw_data
|
|
210
|
-
print("Using a subset of {} samples for network inference.".format(len(self._samp_list)))
|
|
229
|
+
print("Using a ordered subset of {} samples for network inference.".format(len(self._samp_list)))
|
|
211
230
|
|
|
212
231
|
num_sample = len(raw_data)
|
|
213
232
|
params = {'num_epoch': 400,
|
|
@@ -241,7 +260,11 @@ class AbstractDSPIN(ABC):
|
|
|
241
260
|
method: str = 'auto',
|
|
242
261
|
directed: bool = False,
|
|
243
262
|
params: dict = None,
|
|
244
|
-
|
|
263
|
+
sample_list_ordered: List[str] = None,
|
|
264
|
+
prior_network: np.ndarray = None,
|
|
265
|
+
perturb_matrix: np.ndarray = None,
|
|
266
|
+
if_control_key: str = 'if_control',
|
|
267
|
+
batch_key: str = 'batch',
|
|
245
268
|
run_with_matlab: bool = False) -> None:
|
|
246
269
|
"""
|
|
247
270
|
Perform network inference using a specified method and parameters.
|
|
@@ -256,8 +279,16 @@ class AbstractDSPIN(ABC):
|
|
|
256
279
|
Whether to infer a directed network. Default is False.
|
|
257
280
|
params : dict, optional
|
|
258
281
|
Additional parameters for network inference. Default is None.
|
|
259
|
-
|
|
260
|
-
List of
|
|
282
|
+
sample_list_ordered : List[str], optional
|
|
283
|
+
List of ordered sample identifiers. Default is None.
|
|
284
|
+
prior_network : np.ndarray, optional
|
|
285
|
+
Binary matrix of preferred network connectivity from prior knowledge. Default is None.
|
|
286
|
+
perturb_matrix : np.ndarray, optional
|
|
287
|
+
Matrix of putative relative responses to control samples. Default is None.
|
|
288
|
+
if_control_key : str, optional
|
|
289
|
+
Column name in `adata.obs` indicating whether a sample is a control. Default is 'if_control'.
|
|
290
|
+
batch_key : str, optional
|
|
291
|
+
Column name in `adata.obs` specifying batch assignment for each sample. Default is 'batch'.
|
|
261
292
|
run_with_matlab : bool, optional
|
|
262
293
|
If True, prepares data for MATLAB execution instead of running inference in Python. Default is False.
|
|
263
294
|
"""
|
|
@@ -275,8 +306,8 @@ class AbstractDSPIN(ABC):
|
|
|
275
306
|
if method == 'auto':
|
|
276
307
|
samp_list = np.unique(self.adata.obs[sample_id_key])
|
|
277
308
|
num_sample = len(samp_list)
|
|
278
|
-
if
|
|
279
|
-
num_sample = len(
|
|
309
|
+
if sample_list_ordered is not None:
|
|
310
|
+
num_sample = len(sample_list_ordered)
|
|
280
311
|
if num_sample > 30:
|
|
281
312
|
method = 'pseudo_likelihood'
|
|
282
313
|
else:
|
|
@@ -296,9 +327,32 @@ class AbstractDSPIN(ABC):
|
|
|
296
327
|
else:
|
|
297
328
|
self.raw_data_corr(sample_id_key)
|
|
298
329
|
|
|
299
|
-
self.
|
|
330
|
+
self.sample_list_ordered = sample_list_ordered
|
|
300
331
|
|
|
301
332
|
train_dat = self.default_params(method)
|
|
333
|
+
|
|
334
|
+
if prior_network is not None:
|
|
335
|
+
train_dat['j_prior_mask'] = prior_network
|
|
336
|
+
train_dat['lambda_l1_j_prior_mask'] = 0.01
|
|
337
|
+
|
|
338
|
+
if perturb_matrix is not None:
|
|
339
|
+
if if_control_key not in self.adata.obs.columns:
|
|
340
|
+
raise ValueError(f"Column '{if_control_key}' not found in adata.obs.")
|
|
341
|
+
if batch_key not in self.adata.obs.columns:
|
|
342
|
+
raise ValueError(f"Column '{batch_key}' not found in adata.obs.")
|
|
343
|
+
|
|
344
|
+
sample_list = self._samp_list
|
|
345
|
+
sample_to_control = self.adata.obs.groupby(sample_id_key, observed=False)[if_control_key].first().to_dict()
|
|
346
|
+
sample_to_batch = self.adata.obs.groupby(sample_id_key, observed=False)[batch_key].first().to_dict()
|
|
347
|
+
|
|
348
|
+
if_control = np.array([sample_to_control[sample] for sample in sample_list])
|
|
349
|
+
batch_index = np.array([sample_to_batch[sample] for sample in sample_list])
|
|
350
|
+
|
|
351
|
+
train_dat['perturb_matrix'] = perturb_matrix
|
|
352
|
+
train_dat['if_control'] = if_control
|
|
353
|
+
train_dat['batch_index'] = batch_index
|
|
354
|
+
train_dat['lambda_l2_h_rela_prior'] = 0.5
|
|
355
|
+
|
|
302
356
|
train_dat['directed'] = directed
|
|
303
357
|
if params is not None:
|
|
304
358
|
train_dat.update(params)
|
|
@@ -390,9 +444,10 @@ class GeneDSPIN(AbstractDSPIN):
|
|
|
390
444
|
cur_gene = self._onmf_rep_ori[:, ii]
|
|
391
445
|
gene_skew[ii] = skew(cur_gene, bias=False)
|
|
392
446
|
|
|
393
|
-
|
|
447
|
+
print(f'Skewness of the gene expression: {gene_skew.mean().round(3)}')
|
|
448
|
+
|
|
449
|
+
if gene_skew.mean() >= 1.5:
|
|
394
450
|
default_clip = 95
|
|
395
|
-
print("Input gene expression seem to be heavy-tailed. Using 95 percentile for clipping before discretization.")
|
|
396
451
|
else:
|
|
397
452
|
default_clip = 100
|
|
398
453
|
|
|
@@ -400,13 +455,20 @@ class GeneDSPIN(AbstractDSPIN):
|
|
|
400
455
|
'clip_percentile': default_clip,
|
|
401
456
|
'num_init': 10}
|
|
402
457
|
dparams.update(discretize_params)
|
|
403
|
-
|
|
458
|
+
|
|
459
|
+
self.name_list = self.adata.var_names.tolist()
|
|
460
|
+
self.name_list_short = self.adata.var_names.tolist()
|
|
461
|
+
|
|
404
462
|
if dparams['use_default_discretize']:
|
|
463
|
+
|
|
464
|
+
if dparams['clip_percentile'] < 100:
|
|
465
|
+
print(f"Input gene expression seem to be heavy-tailed. Using {dparams['clip_percentile']} percentile for clipping before discretization.")
|
|
466
|
+
|
|
405
467
|
self.discretize(clip_percentile=dparams['clip_percentile'],
|
|
406
468
|
num_init=dparams['num_init'])
|
|
407
469
|
|
|
408
470
|
def program_regulator_discovery(self,
|
|
409
|
-
|
|
471
|
+
program_representation: np.ndarray,
|
|
410
472
|
sample_id_key: str = 'sample_id',
|
|
411
473
|
params: dict = None):
|
|
412
474
|
"""
|
|
@@ -414,7 +476,7 @@ class GeneDSPIN(AbstractDSPIN):
|
|
|
414
476
|
|
|
415
477
|
Parameters
|
|
416
478
|
----------
|
|
417
|
-
|
|
479
|
+
program_representation : np.ndarray
|
|
418
480
|
The gene program representation.
|
|
419
481
|
sample_id_key : str, optional
|
|
420
482
|
Column name in `adata.obs` specifying sample identifiers. Default is 'sample_id'.
|
|
@@ -422,7 +484,7 @@ class GeneDSPIN(AbstractDSPIN):
|
|
|
422
484
|
Additional parameters for regression. Default is None.
|
|
423
485
|
"""
|
|
424
486
|
|
|
425
|
-
program_states, samp_list = sample_states(self.adata.obs[sample_id_key],
|
|
487
|
+
program_states, samp_list = sample_states(self.adata.obs[sample_id_key], program_representation)
|
|
426
488
|
|
|
427
489
|
gene_states, _ = sample_states(self.adata.obs[sample_id_key], self._onmf_rep_tri)
|
|
428
490
|
|
|
@@ -430,7 +492,9 @@ class GeneDSPIN(AbstractDSPIN):
|
|
|
430
492
|
'num_epoch': 500,
|
|
431
493
|
'stepsz': 0.01,
|
|
432
494
|
'lambda_l1_interaction': 0.01,
|
|
433
|
-
'rec_gap': 10
|
|
495
|
+
'rec_gap': 10,
|
|
496
|
+
'backtrack_gap': 40,
|
|
497
|
+
'backtrack_tol': 5
|
|
434
498
|
}
|
|
435
499
|
|
|
436
500
|
if params is not None:
|
|
@@ -664,11 +728,11 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
664
728
|
self.prior_programs_ind = prior_program_ind
|
|
665
729
|
|
|
666
730
|
num_onmf_components = onmf_parameters['num_onmf_components']
|
|
731
|
+
|
|
732
|
+
os.makedirs(self.save_path + 'onmf/', exist_ok=True)
|
|
667
733
|
|
|
668
734
|
if mode == 'compute_summary' or mode == 'compute_only':
|
|
669
735
|
|
|
670
|
-
os.makedirs(self.save_path + 'onmf/', exist_ok=True)
|
|
671
|
-
|
|
672
736
|
cur_gene_matrix = adata.X[:, prior_program_mask]
|
|
673
737
|
cluster_label_raw = adata.obs[cluster_key].values
|
|
674
738
|
cluster_label = pd.factorize(cluster_label_raw)[0]
|
|
@@ -709,10 +773,11 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
709
773
|
# Save the gene programs to a CSV file
|
|
710
774
|
file_path = onmf_to_csv(onmf_summary.components_, adata.var_names, self.save_path + f'gene_programs_total_{num_spin}_onmf_{num_onmf_components}_prior_{len(prior_program_ind)}.csv', thres=0.01)
|
|
711
775
|
print('Gene program information saved to {}'.format(file_path))
|
|
712
|
-
file_wpath = onmf_to_csv(onmf_summary.components_, adata.var_names, self.save_path + f'gene_programs_total_{num_spin}_onmf_{num_onmf_components}_prior_{len(prior_program_ind)}_weights.csv', thres=0.01, if_write_weights=True)
|
|
713
|
-
# print('Gene programs saved to {}'.format(file_path))
|
|
714
776
|
self.gene_program_csv = file_path
|
|
715
777
|
|
|
778
|
+
file_wpath = onmf_to_csv(onmf_summary.components_, adata.var_names, self.save_path + f'gene_programs_total_{num_spin}_onmf_{num_onmf_components}_prior_{len(prior_program_ind)}_weights.csv', thres=0.01, if_write_weights=True)
|
|
779
|
+
# print('Gene programs saved to {}'.format(file_path))
|
|
780
|
+
|
|
716
781
|
gene_matrix = self.adata.X
|
|
717
782
|
if issparse(gene_matrix):
|
|
718
783
|
gene_matrix = np.asarray(gene_matrix.toarray()).astype(np.float64)
|
|
@@ -727,6 +792,20 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
727
792
|
self.discretize(clip_percentile=dparams['clip_percentile'],
|
|
728
793
|
num_init=dparams['num_init'])
|
|
729
794
|
|
|
795
|
+
num_display_gene = 5
|
|
796
|
+
gene_pd = pd.read_csv(self.gene_program_csv, index_col=None)
|
|
797
|
+
|
|
798
|
+
top_gene_list = [gene_pd.iloc[:, ii].tolist()[:num_display_gene] for ii in range(num_spin)]
|
|
799
|
+
# remove nan
|
|
800
|
+
top_gene_list_filtered = []
|
|
801
|
+
for sub_list in top_gene_list:
|
|
802
|
+
sub_list_filtered = [gene for gene in sub_list if str(gene) != 'nan']
|
|
803
|
+
top_gene_list_filtered.append(sub_list_filtered)
|
|
804
|
+
|
|
805
|
+
program_names = [f'P{ii}-' + ','.join(top_gene_list_filtered[ii]) for ii in range(num_spin)]
|
|
806
|
+
self.name_list = program_names
|
|
807
|
+
self.name_list_short = [name.split('-')[0] for name in program_names]
|
|
808
|
+
|
|
730
809
|
|
|
731
810
|
class DSPIN(object):
|
|
732
811
|
"""
|