dspin 1.1.2__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dspin-1.1.2/dspin.egg-info → dspin-1.2.0}/PKG-INFO +1 -1
- {dspin-1.1.2 → dspin-1.2.0}/dspin/annotate.py +17 -16
- {dspin-1.1.2 → dspin-1.2.0}/dspin/compute.py +170 -11
- {dspin-1.1.2 → dspin-1.2.0}/dspin/dspin.py +305 -236
- {dspin-1.1.2 → dspin-1.2.0/dspin.egg-info}/PKG-INFO +1 -1
- {dspin-1.1.2 → dspin-1.2.0}/setup.py +1 -1
- {dspin-1.1.2 → dspin-1.2.0}/LICENSE +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/README.md +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/__pycache__/__init__.cpython-310.pyc +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/__pycache__/__init__.cpython-37.pyc +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/__pycache__/compute.cpython-310.pyc +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/__pycache__/compute.cpython-37.pyc +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/__pycache__/dspin_abstract.cpython-310.pyc +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/__pycache__/plot.cpython-310.pyc +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/__pycache__/plotting.cpython-37.pyc +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/dspin/__init__.py +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/dspin/plot.py +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/dspin.egg-info/SOURCES.txt +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/dspin.egg-info/dependency_links.txt +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/dspin.egg-info/requires.txt +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/dspin.egg-info/top_level.txt +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/requirements.txt +0 -0
- {dspin-1.1.2 → dspin-1.2.0}/setup.cfg +0 -0
|
@@ -82,9 +82,9 @@ def automatic_david_annotation(data_folder, file_name, token_name, species='huma
|
|
|
82
82
|
|
|
83
83
|
# Save results to Excel
|
|
84
84
|
save_to_excel(data_folder, file_name,
|
|
85
|
-
|
|
85
|
+
gene_id_map_reverse, all_onmf_df.columns.tolist(), full=False)
|
|
86
86
|
save_to_excel(data_folder, file_name,
|
|
87
|
-
|
|
87
|
+
gene_id_map_reverse, all_onmf_df.columns.tolist(), full=True)
|
|
88
88
|
|
|
89
89
|
|
|
90
90
|
def make_user_prompt_with_score(genes, feature_df=[], direct=False, customized_prompt=None):
|
|
@@ -285,10 +285,10 @@ def setup_david_client(token_name):
|
|
|
285
285
|
|
|
286
286
|
from suds.client import Client
|
|
287
287
|
|
|
288
|
-
url = 'https://
|
|
288
|
+
url = 'https://davidbioinformatics.nih.gov/webservice/services/DAVIDWebService?wsdl'
|
|
289
289
|
client = Client(url)
|
|
290
290
|
client.wsdl.services[0].setlocation(
|
|
291
|
-
'https://
|
|
291
|
+
'https://davidbioinformatics.nih.gov/webservice/services/DAVIDWebService.DAVIDWebServiceHttpSoap11Endpoint/')
|
|
292
292
|
client.service.authenticate(token_name)
|
|
293
293
|
return client
|
|
294
294
|
|
|
@@ -359,7 +359,7 @@ def process_gene_lists(data_folder, file_name, all_onmf_df, gene_id_map, client)
|
|
|
359
359
|
for i in range(num_lists):
|
|
360
360
|
cur_gene_list = all_onmf_df.iloc[:, i].dropna()
|
|
361
361
|
cur_gene_list = map_gene_ids(gene_id_map, cur_gene_list)
|
|
362
|
-
save_path = os.path.join(save_folder, f'list_{i}.csv')
|
|
362
|
+
save_path = os.path.join(save_folder, f'list_{all_onmf_df.columns[i]}.csv')
|
|
363
363
|
|
|
364
364
|
input_ids = ','.join(cur_gene_list)
|
|
365
365
|
id_type = 'ENSEMBL_GENE_ID'
|
|
@@ -370,7 +370,7 @@ def process_gene_lists(data_folder, file_name, all_onmf_df, gene_id_map, client)
|
|
|
370
370
|
term_clustering_report = client.service.getTermClusterReport(
|
|
371
371
|
overlap, initialSeed, finalSeed, linkage, kappa)
|
|
372
372
|
|
|
373
|
-
print(f'Gene list {i} ', end='\t')
|
|
373
|
+
print(f'Gene list {all_onmf_df.columns[i]} ', end='\t')
|
|
374
374
|
# print(term_clustering_report)
|
|
375
375
|
if term_clustering_report is None:
|
|
376
376
|
raise Exception(
|
|
@@ -379,7 +379,7 @@ def process_gene_lists(data_folder, file_name, all_onmf_df, gene_id_map, client)
|
|
|
379
379
|
parse_david_output(term_clustering_report, save_path)
|
|
380
380
|
|
|
381
381
|
|
|
382
|
-
def save_to_excel(data_folder, file_name, gene_id_map_reverse,
|
|
382
|
+
def save_to_excel(data_folder, file_name, gene_id_map_reverse, list_name, full=True):
|
|
383
383
|
"""
|
|
384
384
|
Save the parsed DAVID output to an Excel file, with an option to include full or filtered results.
|
|
385
385
|
|
|
@@ -387,7 +387,7 @@ def save_to_excel(data_folder, file_name, gene_id_map_reverse, num_lists, full=T
|
|
|
387
387
|
data_folder (str): The folder where the data files are located.
|
|
388
388
|
file_name (str): The name of the original CSV file containing gene data.
|
|
389
389
|
gene_id_map_reverse (dict): A dictionary mapping ENSEMBL IDs to gene symbols.
|
|
390
|
-
|
|
390
|
+
list_name (list): A list of gene list names.
|
|
391
391
|
full (bool): Whether to save the full results. If False, only results with Pvalue <= 0.05 are saved.
|
|
392
392
|
"""
|
|
393
393
|
|
|
@@ -433,8 +433,8 @@ def save_to_excel(data_folder, file_name, gene_id_map_reverse, num_lists, full=T
|
|
|
433
433
|
'.csv', '_david_full.xlsx') if full else file_name.replace('.csv', '_david.xlsx')
|
|
434
434
|
|
|
435
435
|
with pd.ExcelWriter(data_folder + xlsx_name) as writer:
|
|
436
|
-
for
|
|
437
|
-
cur_pd = pd.read_csv(save_folder + 'list_
|
|
436
|
+
for lname in list_name:
|
|
437
|
+
cur_pd = pd.read_csv(save_folder + f'list_{lname}.csv', sep='\t', names=[
|
|
438
438
|
'Category', 'Term', 'Count', '%', 'Pvalue', 'Genes', 'List Total', 'Pop Hits', 'Pop Total', 'Fold Enrichment', 'Bonferroni', 'Benjamini', 'FDR'])
|
|
439
439
|
filt1 = cur_pd['Category'] == 'UP_SEQ_FEATURE'
|
|
440
440
|
filt2 = cur_pd['Pvalue'].apply(filt_fun)
|
|
@@ -453,9 +453,10 @@ def save_to_excel(data_folder, file_name, gene_id_map_reverse, num_lists, full=T
|
|
|
453
453
|
cur_pd = pd.DataFrame(np.insert(
|
|
454
454
|
cur_pd.values, ind + kk, values=[" "] * len(cur_pd.columns), axis=0), columns=cur_pd.columns)
|
|
455
455
|
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
writer.sheets[
|
|
459
|
-
writer.sheets[
|
|
460
|
-
writer.sheets[
|
|
461
|
-
writer.sheets[
|
|
456
|
+
sheet_name = lname
|
|
457
|
+
cur_pd.to_excel(writer, sheet_name=sheet_name, index=False)
|
|
458
|
+
writer.sheets[sheet_name].set_column(0, 0, 25)
|
|
459
|
+
writer.sheets[sheet_name].set_column(1, 1, 50)
|
|
460
|
+
writer.sheets[sheet_name].set_column(2, 2, 5)
|
|
461
|
+
writer.sheets[sheet_name].set_column(3, 3, 5)
|
|
462
|
+
writer.sheets[sheet_name].set_column(5, 5, 100)
|
|
@@ -17,7 +17,7 @@ import os
|
|
|
17
17
|
from sklearn.cluster import KMeans
|
|
18
18
|
import scipy.io as sio
|
|
19
19
|
from tqdm import tqdm
|
|
20
|
-
from typing import List
|
|
20
|
+
from typing import List, Tuple
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def category_balance_number(
|
|
@@ -209,14 +209,14 @@ def onmf(X: np.array, rank: int, max_iter: int = 200) -> (np.array, np.array):
|
|
|
209
209
|
return S.T, A
|
|
210
210
|
|
|
211
211
|
|
|
212
|
-
def compute_onmf(seed: int, num_spin: int,
|
|
212
|
+
def compute_onmf(seed: int, num_spin: int, gene_matrix: np.array) -> NMF:
|
|
213
213
|
"""
|
|
214
214
|
Computes the ONMF model for the given gene matrix.
|
|
215
215
|
|
|
216
216
|
Parameters:
|
|
217
217
|
seed (int): Seed for Random Number Generation.
|
|
218
218
|
num_spin (int): The number of desired components (clusters/spins).
|
|
219
|
-
|
|
219
|
+
gene_matrix (np.array): Matrix representing the gene expression data. Typically the matrix should be normalized by library size and log-1 transfromed. Normalize each gene by its standard devaition is also recommended.
|
|
220
220
|
|
|
221
221
|
Returns:
|
|
222
222
|
NMF: The NMF model with computed components.
|
|
@@ -225,7 +225,7 @@ def compute_onmf(seed: int, num_spin: int, gene_matrix_bin: np.array) -> NMF:
|
|
|
225
225
|
# Generate a random seed
|
|
226
226
|
np.random.seed(seed)
|
|
227
227
|
# Factorized Matrices
|
|
228
|
-
H, W = onmf(
|
|
228
|
+
H, W = onmf(gene_matrix, num_spin)
|
|
229
229
|
|
|
230
230
|
# Initialize the NMF model
|
|
231
231
|
nmf_model = NMF(n_components=num_spin, random_state=seed)
|
|
@@ -338,6 +338,18 @@ def sample_corr_mean(samp_full: np.array,
|
|
|
338
338
|
return raw_corr_data, samp_list
|
|
339
339
|
|
|
340
340
|
|
|
341
|
+
def sample_states(samp_full: np.array, onmf_rep_tri: np.array) -> Tuple[np.array, np.array]:
|
|
342
|
+
|
|
343
|
+
samp_list = np.unique(samp_full)
|
|
344
|
+
state_list = np.zeros(len(samp_list), dtype=object)
|
|
345
|
+
|
|
346
|
+
for ii, cur_samp in enumerate(samp_list):
|
|
347
|
+
cur_filt = samp_full == cur_samp
|
|
348
|
+
cur_state = onmf_rep_tri[cur_filt, :]
|
|
349
|
+
state_list[ii] = cur_state.T
|
|
350
|
+
|
|
351
|
+
return state_list, samp_list
|
|
352
|
+
|
|
341
353
|
def para_moments(j_mat: np.array, h_vec: np.array) -> (np.array, np.array):
|
|
342
354
|
"""
|
|
343
355
|
Calculates the mean and correlation given j network and h vectors.
|
|
@@ -805,7 +817,7 @@ def learn_network_adam(raw_data, method, train_dat):
|
|
|
805
817
|
# during training
|
|
806
818
|
trace_epoch = 50 + num_epoch - counter
|
|
807
819
|
pos = num_epoch - trace_epoch + np.argmin(rec_jgrad_sum_norm[- trace_epoch: ])
|
|
808
|
-
print(pos)
|
|
820
|
+
# print(pos)
|
|
809
821
|
cur_h = rec_hvec_all[pos, :, :]
|
|
810
822
|
cur_j = rec_jmat_all[pos, :, :]
|
|
811
823
|
|
|
@@ -815,6 +827,151 @@ def learn_network_adam(raw_data, method, train_dat):
|
|
|
815
827
|
return cur_j, cur_h, train_log
|
|
816
828
|
|
|
817
829
|
|
|
830
|
+
|
|
831
|
+
def learn_program_regulators(gene_states, program_states, train_dat):
|
|
832
|
+
"""
|
|
833
|
+
Discover regulators for gene programs via regression.
|
|
834
|
+
|
|
835
|
+
Parameters
|
|
836
|
+
----------
|
|
837
|
+
gene_states : list of np.ndarray
|
|
838
|
+
Each element is a gene state array of shape (n_spin, T),
|
|
839
|
+
where T may vary across samples.
|
|
840
|
+
program_states : list of np.ndarray
|
|
841
|
+
Each element is a program state array of shape (n_target, T).
|
|
842
|
+
train_dat : dict
|
|
843
|
+
Dictionary of training parameters. Expected keys (with defaults):
|
|
844
|
+
- 'num_epoch': int, default 300
|
|
845
|
+
- 'stepsz': float, default 0.01
|
|
846
|
+
- 'decay': float, default 0.2 (not explicitly used here)
|
|
847
|
+
- 'interaction_l1': float, default 0.01
|
|
848
|
+
|
|
849
|
+
Returns
|
|
850
|
+
-------
|
|
851
|
+
cur_interaction : np.ndarray
|
|
852
|
+
Learned interaction matrix of shape (n_spin, n_target).
|
|
853
|
+
cur_selfj : np.ndarray
|
|
854
|
+
Learned self-interaction (regulator-specific bias) of shape (n_target,).
|
|
855
|
+
cur_selfh : np.ndarray
|
|
856
|
+
Learned program activity bias of shape (n_target,).
|
|
857
|
+
"""
|
|
858
|
+
|
|
859
|
+
num_gene = gene_states[0].shape[0]
|
|
860
|
+
num_program = program_states[0].shape[0]
|
|
861
|
+
num_round = len(gene_states)
|
|
862
|
+
|
|
863
|
+
num_epoch, stepsz, rec_gap = (
|
|
864
|
+
train_dat.get(
|
|
865
|
+
key, None) for key in [
|
|
866
|
+
"num_epoch", "stepsz", "rec_gap"])
|
|
867
|
+
list_step = np.arange(num_epoch, 0, - rec_gap)[::-1]
|
|
868
|
+
|
|
869
|
+
save_path = train_dat.get('save_path', None)
|
|
870
|
+
|
|
871
|
+
backtrack_gap, backtrack_tol = (
|
|
872
|
+
train_dat.get(
|
|
873
|
+
key, None) for key in [
|
|
874
|
+
"backtrack_gap", "backtrack_tol"])
|
|
875
|
+
|
|
876
|
+
lambda_l1_j, lambda_l2_j = (train_dat.get(key, 0) for key in ["lambda_l1_j", "lambda_l2_j"])
|
|
877
|
+
|
|
878
|
+
# Initialize parameters (using zeros)
|
|
879
|
+
cur_interaction = np.zeros((num_gene, num_program))
|
|
880
|
+
cur_selfj = np.zeros(num_program)
|
|
881
|
+
cur_selfh = np.zeros(num_program)
|
|
882
|
+
|
|
883
|
+
# Compute sample weights based on the number of states (columns) in each gene_states sample
|
|
884
|
+
state_sizes = np.array([state.shape[1] for state in gene_states])
|
|
885
|
+
samp_weight = np.sqrt(state_sizes)
|
|
886
|
+
samp_weight = samp_weight / np.sum(samp_weight)
|
|
887
|
+
|
|
888
|
+
# Preallocate gradient norm recording (optional)
|
|
889
|
+
rec_grad = np.zeros(num_epoch)
|
|
890
|
+
|
|
891
|
+
m_interaction, v_interaction = np.zeros_like(cur_interaction), np.zeros_like(cur_interaction)
|
|
892
|
+
m_selfj, v_selfj = np.zeros_like(cur_selfj), np.zeros_like(cur_selfj)
|
|
893
|
+
m_selfh, v_selfh = np.zeros_like(cur_selfh), np.zeros_like(cur_selfh)
|
|
894
|
+
|
|
895
|
+
# Create l1 regularization schedule
|
|
896
|
+
l1_base_start = 0.02
|
|
897
|
+
l1_base_end = 1e-3
|
|
898
|
+
|
|
899
|
+
part1 = np.full((num_epoch // 4 + 1,), l1_base_start)
|
|
900
|
+
part2 = np.logspace(np.log10(l1_base_start), np.log10(l1_base_end), num=num_epoch // 2)
|
|
901
|
+
part3 = np.full((num_epoch // 4 + 1,), l1_base_end)
|
|
902
|
+
l1_base_array = np.concatenate([part1, part2, part3])[:num_epoch]
|
|
903
|
+
|
|
904
|
+
# Preallocate arrays to collect per-sample gradients
|
|
905
|
+
rec_interaction_grad = np.zeros((num_round, num_gene, num_program))
|
|
906
|
+
rec_selfj_grad = np.zeros((num_round, num_program))
|
|
907
|
+
rec_selfh_grad = np.zeros((num_round, num_program))
|
|
908
|
+
|
|
909
|
+
pbar = tqdm(total=num_epoch)
|
|
910
|
+
counter = 1
|
|
911
|
+
|
|
912
|
+
while counter <= num_epoch:
|
|
913
|
+
|
|
914
|
+
l1_base = l1_base_array[counter - 1]
|
|
915
|
+
|
|
916
|
+
for kk in range(num_round):
|
|
917
|
+
cur_state = gene_states[kk]
|
|
918
|
+
cur_target = program_states[kk]
|
|
919
|
+
|
|
920
|
+
effective_h = cur_state.T @ cur_interaction + cur_selfh.T
|
|
921
|
+
|
|
922
|
+
# j_sub is the bias for regulators (broadcasted as row vector)
|
|
923
|
+
j_sub = cur_selfj.T # shape: (n_target,)
|
|
924
|
+
|
|
925
|
+
# Compute terms.
|
|
926
|
+
# We follow MATLAB: term1 = exp(j_sub + effective_h)' so that term1 becomes (n_target, T)
|
|
927
|
+
term1 = np.exp(j_sub + effective_h).T
|
|
928
|
+
term2 = np.exp(j_sub - effective_h).T
|
|
929
|
+
|
|
930
|
+
# Compute gradients for the bias parameters per sample
|
|
931
|
+
j_sub_grad = cur_target**2 - (term1 + term2) / (term1 + term2 + 1)
|
|
932
|
+
h_eff_grad = cur_target - (term1 - term2) / (term1 + term2 + 1)
|
|
933
|
+
|
|
934
|
+
# Gradient for interaction parameters
|
|
935
|
+
# cur_state: (n_spin, T), h_eff_grad.T: (T, n_target)
|
|
936
|
+
j_off_sub_grad = cur_state @ h_eff_grad.T # shape: (n_spin, n_target)
|
|
937
|
+
|
|
938
|
+
# Average over the T (state) dimension
|
|
939
|
+
rec_interaction_grad[kk] = j_off_sub_grad / cur_state.shape[1]
|
|
940
|
+
rec_selfj_grad[kk] = np.mean(j_sub_grad, axis=1)
|
|
941
|
+
rec_selfh_grad[kk] = np.mean(h_eff_grad, axis=1)
|
|
942
|
+
|
|
943
|
+
# Aggregate gradients across samples using sample weights
|
|
944
|
+
interaction_grad = - np.sum(rec_interaction_grad * samp_weight[:, None, None], axis=0)
|
|
945
|
+
selfj_grad = - np.sum(rec_selfj_grad * samp_weight[:, None], axis=0)
|
|
946
|
+
selfh_grad = - np.sum(rec_selfh_grad * samp_weight[:, None], axis=0)
|
|
947
|
+
|
|
948
|
+
if lambda_l1_j > 0:
|
|
949
|
+
interaction_grad += lambda_l1_j * (cur_interaction / l1_base).clip(- 1, 1)
|
|
950
|
+
if lambda_l2_j > 0:
|
|
951
|
+
interaction_grad += lambda_l2_j * cur_interaction
|
|
952
|
+
|
|
953
|
+
# Update parameters using the provided update_adam function
|
|
954
|
+
update_val, m_interaction, v_interaction = update_adam(interaction_grad, m_interaction, v_interaction, counter, stepsz)
|
|
955
|
+
cur_interaction = cur_interaction - update_val
|
|
956
|
+
|
|
957
|
+
update_val, m_selfj, v_selfj = update_adam(selfj_grad, m_selfj, v_selfj, counter, stepsz)
|
|
958
|
+
cur_selfj = cur_selfj - update_val
|
|
959
|
+
|
|
960
|
+
update_val, m_selfh, v_selfh = update_adam(selfh_grad, m_selfh, v_selfh, counter, stepsz)
|
|
961
|
+
cur_selfh = cur_selfh - update_val
|
|
962
|
+
|
|
963
|
+
# Record gradient norm (optional)
|
|
964
|
+
rec_grad[counter - 1] = np.linalg.norm(interaction_grad)
|
|
965
|
+
|
|
966
|
+
if counter in list_step:
|
|
967
|
+
pbar.update(rec_gap)
|
|
968
|
+
pbar.set_postfix({"Network Gradient": f"{rec_grad[counter - 1]:.4f}"})
|
|
969
|
+
|
|
970
|
+
counter += 1
|
|
971
|
+
|
|
972
|
+
return cur_interaction, cur_selfj, cur_selfh
|
|
973
|
+
|
|
974
|
+
|
|
818
975
|
def compute_relative_responses(cur_h, if_control, batch_index):
|
|
819
976
|
"""
|
|
820
977
|
Compute the relative responses of samples by subtracting the average of control samples.
|
|
@@ -831,20 +988,22 @@ def compute_relative_responses(cur_h, if_control, batch_index):
|
|
|
831
988
|
numpy.ndarray: A 2D array of the same shape as `cur_h`, containing the relative responses for each sample.
|
|
832
989
|
"""
|
|
833
990
|
|
|
834
|
-
if np.all(if_control == 0):
|
|
835
|
-
raise ValueError('if_control contains all zeros')
|
|
836
991
|
|
|
837
992
|
unique_batches = np.unique(batch_index)
|
|
838
993
|
num_batches = len(unique_batches)
|
|
839
994
|
|
|
840
995
|
relative_h = np.zeros_like(cur_h)
|
|
841
|
-
|
|
996
|
+
|
|
997
|
+
if np.all(if_control == 0):
|
|
998
|
+
all_controls_mean = np.mean(cur_h, axis=1)
|
|
999
|
+
else:
|
|
1000
|
+
all_controls_mean = np.mean(cur_h[:, if_control], axis=1)
|
|
842
1001
|
|
|
843
1002
|
for current_batch in unique_batches:
|
|
844
1003
|
batch_samples_idx = np.where(batch_index == current_batch)[0]
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
1004
|
+
|
|
1005
|
+
if np.any(if_control[batch_samples_idx]):
|
|
1006
|
+
batch_controls_idx = batch_samples_idx[if_control[batch_samples_idx]]
|
|
848
1007
|
batch_controls_mean = np.mean(cur_h[:, batch_controls_idx], axis=1)
|
|
849
1008
|
else:
|
|
850
1009
|
batch_controls_mean = all_controls_mean
|
|
@@ -4,13 +4,13 @@
|
|
|
4
4
|
@Author : Jialong Jiang, Yingying Gong
|
|
5
5
|
'''
|
|
6
6
|
|
|
7
|
-
from .plot import
|
|
8
|
-
onmf_to_csv
|
|
9
|
-
)
|
|
7
|
+
from .plot import onmf_to_csv
|
|
10
8
|
from .compute import (
|
|
11
9
|
onmf_discretize,
|
|
12
10
|
sample_corr_mean,
|
|
11
|
+
sample_states,
|
|
13
12
|
learn_network_adam,
|
|
13
|
+
learn_program_regulators,
|
|
14
14
|
category_balance_number,
|
|
15
15
|
compute_onmf,
|
|
16
16
|
summary_components,
|
|
@@ -32,36 +32,22 @@ import networkx as nx
|
|
|
32
32
|
import matplotlib.patheffects as patheffects
|
|
33
33
|
import warnings
|
|
34
34
|
import itertools
|
|
35
|
-
from typing import List
|
|
35
|
+
from typing import List, Dict, Optional, Tuple, Union
|
|
36
|
+
|
|
36
37
|
|
|
37
38
|
|
|
38
39
|
class AbstractDSPIN(ABC):
|
|
39
40
|
"""
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
...
|
|
41
|
+
Abstract base class for DSPIN, providing common methods and properties for GeneDSPIN and ProgramDSPIN.
|
|
42
|
+
|
|
44
43
|
Attributes
|
|
45
44
|
----------
|
|
46
45
|
adata : ad.AnnData
|
|
47
|
-
Annotated data.
|
|
46
|
+
Annotated single-cell gene expression data.
|
|
48
47
|
save_path : str
|
|
49
|
-
|
|
48
|
+
Directory path where results will be stored.
|
|
50
49
|
num_spin : int
|
|
51
|
-
Number of spins.
|
|
52
|
-
|
|
53
|
-
Methods
|
|
54
|
-
-------
|
|
55
|
-
discretize()
|
|
56
|
-
Discretizes the ONMF representation into three states (-1, 0, 1) using K-means clustering.
|
|
57
|
-
raw_data_corr(sample_col_name)
|
|
58
|
-
Computes correlation of raw data based on sample column name.
|
|
59
|
-
raw_data_state(sample_col_name)
|
|
60
|
-
Calculate and return the correlation of raw data.
|
|
61
|
-
default_params(method)
|
|
62
|
-
Provide the default parameters for the specified algorithm.
|
|
63
|
-
network_inference(sample_col_name, method, params, example_list, record_step)
|
|
64
|
-
Execute the network inference using the specified method and parameters and record the results.
|
|
50
|
+
Number of spins for modeling.
|
|
65
51
|
"""
|
|
66
52
|
|
|
67
53
|
def __init__(self,
|
|
@@ -69,12 +55,16 @@ class AbstractDSPIN(ABC):
|
|
|
69
55
|
save_path: str,
|
|
70
56
|
num_spin: int):
|
|
71
57
|
"""
|
|
72
|
-
Initialize
|
|
73
|
-
|
|
74
|
-
Parameters
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
58
|
+
Initialize an instance of AbstractDSPIN.
|
|
59
|
+
|
|
60
|
+
Parameters
|
|
61
|
+
----------
|
|
62
|
+
adata : ad.AnnData
|
|
63
|
+
Annotated data matrix containing observations and gene expressions.
|
|
64
|
+
save_path : str
|
|
65
|
+
Directory path for saving results.
|
|
66
|
+
num_spin : int
|
|
67
|
+
Number of spins used in DSPIN.
|
|
78
68
|
"""
|
|
79
69
|
|
|
80
70
|
self.adata = adata
|
|
@@ -84,15 +74,16 @@ class AbstractDSPIN(ABC):
|
|
|
84
74
|
if not os.path.exists(self.save_path):
|
|
85
75
|
os.makedirs(self.save_path)
|
|
86
76
|
print("Saving path does not exist. Creating a new folder.")
|
|
87
|
-
|
|
77
|
+
|
|
78
|
+
self.fig_folder = self.save_path + 'figures/'
|
|
88
79
|
os.makedirs(self.fig_folder, exist_ok=True)
|
|
89
80
|
|
|
90
81
|
@property
|
|
91
|
-
def
|
|
82
|
+
def program_representation_raw(self):
|
|
92
83
|
return self._onmf_rep_ori
|
|
93
84
|
|
|
94
85
|
@property
|
|
95
|
-
def
|
|
86
|
+
def program_representation(self):
|
|
96
87
|
return self._onmf_rep_tri
|
|
97
88
|
|
|
98
89
|
@property
|
|
@@ -115,12 +106,12 @@ class AbstractDSPIN(ABC):
|
|
|
115
106
|
def sample_list(self):
|
|
116
107
|
return self._samp_list
|
|
117
108
|
|
|
118
|
-
@
|
|
119
|
-
def
|
|
109
|
+
@program_representation_raw.setter
|
|
110
|
+
def program_representation_raw(self, value):
|
|
120
111
|
self._onmf_rep_ori = value
|
|
121
112
|
|
|
122
|
-
@
|
|
123
|
-
def
|
|
113
|
+
@program_representation.setter
|
|
114
|
+
def program_representation(self, value):
|
|
124
115
|
self._onmf_rep_tri = value
|
|
125
116
|
|
|
126
117
|
@network.setter
|
|
@@ -139,53 +130,52 @@ class AbstractDSPIN(ABC):
|
|
|
139
130
|
def sample_list(self, value):
|
|
140
131
|
self._samp_list = value
|
|
141
132
|
|
|
142
|
-
def discretize(self) ->
|
|
133
|
+
def discretize(self, clip_percentile: float = 100) -> None:
|
|
143
134
|
"""
|
|
144
|
-
Discretizes the
|
|
135
|
+
Discretizes the oNMF representation into three states (-1, 0, 1) using K-means clustering.
|
|
145
136
|
|
|
146
|
-
|
|
147
|
-
|
|
137
|
+
Parameters
|
|
138
|
+
----------
|
|
139
|
+
clip_percentile : float, optional
|
|
140
|
+
Percentile at which to clip values before discretization (default is 100).
|
|
148
141
|
"""
|
|
149
|
-
onmf_rep_ori = self.
|
|
142
|
+
onmf_rep_ori = self.program_representation_raw.copy()
|
|
150
143
|
fig_folder = self.fig_folder
|
|
151
144
|
|
|
145
|
+
if clip_percentile < 100:
|
|
146
|
+
for ii in range(onmf_rep_ori.shape[1]):
|
|
147
|
+
onmf_rep_ori[:, ii] = onmf_rep_ori[:, ii] / np.percentile(onmf_rep_ori[:, ii], clip_percentile)
|
|
148
|
+
onmf_rep_ori = onmf_rep_ori.clip(0, 1)
|
|
149
|
+
|
|
152
150
|
onmf_rep_tri = onmf_discretize(onmf_rep_ori, fig_folder)
|
|
153
151
|
self._onmf_rep_tri = onmf_rep_tri
|
|
154
152
|
|
|
155
|
-
def raw_data_corr(self,
|
|
153
|
+
def raw_data_corr(self, sample_id_key: str) -> None:
|
|
156
154
|
"""
|
|
157
|
-
Computes correlation of raw data based on sample column name.
|
|
155
|
+
Computes correlation of raw data based on a sample column name.
|
|
158
156
|
|
|
159
|
-
|
|
160
|
-
|
|
157
|
+
Parameters
|
|
158
|
+
----------
|
|
159
|
+
sample_id_key : str
|
|
160
|
+
Column name in adata.obs specifying sample identifiers.
|
|
161
161
|
"""
|
|
162
|
-
|
|
163
162
|
raw_data, samp_list = sample_corr_mean(
|
|
164
|
-
self.adata.obs[
|
|
163
|
+
self.adata.obs[sample_id_key], self._onmf_rep_tri)
|
|
165
164
|
self._raw_data = raw_data
|
|
166
165
|
self._samp_list = samp_list
|
|
167
166
|
|
|
168
|
-
def raw_data_state(self,
|
|
167
|
+
def raw_data_state(self, sample_id_key) -> None:
|
|
169
168
|
"""
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
Parameters:
|
|
173
|
-
sample_col_name (str): The name of the column in the sample to calculate the correlation.
|
|
169
|
+
Compute and return the state of raw data based on sample identifiers.
|
|
174
170
|
|
|
175
|
-
|
|
176
|
-
|
|
171
|
+
Parameters
|
|
172
|
+
----------
|
|
173
|
+
sample_id_key : str
|
|
174
|
+
Column name in `adata.obs` used to group and compute raw data states.
|
|
177
175
|
"""
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
samp_list = np.unique(cadata.obs[sample_col_name])
|
|
182
|
-
state_list = np.zeros(len(samp_list), dtype=object)
|
|
183
|
-
|
|
184
|
-
for ii, cur_samp in enumerate(samp_list):
|
|
185
|
-
cur_filt = cadata.obs[sample_col_name] == cur_samp
|
|
186
|
-
cur_state = self._onmf_rep_tri[cur_filt, :]
|
|
187
|
-
state_list[ii] = cur_state.T
|
|
188
|
-
|
|
176
|
+
state_list, samp_list = sample_states(
|
|
177
|
+
self.adata.obs[sample_id_key], self._onmf_rep_tri)
|
|
178
|
+
|
|
189
179
|
self._raw_data = state_list
|
|
190
180
|
self._samp_list = samp_list
|
|
191
181
|
|
|
@@ -194,8 +184,10 @@ class AbstractDSPIN(ABC):
|
|
|
194
184
|
"""
|
|
195
185
|
Provide the default parameters for the specified algorithm.
|
|
196
186
|
|
|
197
|
-
Parameters
|
|
198
|
-
|
|
187
|
+
Parameters
|
|
188
|
+
----------
|
|
189
|
+
method : str
|
|
190
|
+
The inference method. Options: 'maximum_likelihood', 'mcmc_maximum_likelihood', 'pseudo_likelihood'.
|
|
199
191
|
|
|
200
192
|
Returns:
|
|
201
193
|
dict: Dictionary containing default parameters for the specified method.
|
|
@@ -215,7 +207,8 @@ class AbstractDSPIN(ABC):
|
|
|
215
207
|
params = {'num_epoch': 200,
|
|
216
208
|
'cur_j': np.zeros((num_spin, num_spin)),
|
|
217
209
|
'cur_h': np.zeros((num_spin, num_sample)),
|
|
218
|
-
'save_path': self.save_path
|
|
210
|
+
'save_path': self.save_path,
|
|
211
|
+
'rec_gap': 10}
|
|
219
212
|
params.update({'lambda_l1_j': 0.01,
|
|
220
213
|
'lambda_l1_h': 0,
|
|
221
214
|
'lambda_l2_j': 0,
|
|
@@ -236,29 +229,32 @@ class AbstractDSPIN(ABC):
|
|
|
236
229
|
return params
|
|
237
230
|
|
|
238
231
|
def network_inference(self,
|
|
239
|
-
|
|
232
|
+
sample_id_key: str = 'sample_id',
|
|
240
233
|
method: str = 'auto',
|
|
241
234
|
directed: bool = False,
|
|
242
235
|
params: dict = None,
|
|
243
236
|
example_list: List[str] = None,
|
|
244
|
-
|
|
245
|
-
run_with_matlab: bool = False,
|
|
246
|
-
precomputed_discretization: np.array = None):
|
|
237
|
+
run_with_matlab: bool = False) -> None:
|
|
247
238
|
"""
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
Parameters
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
239
|
+
Perform network inference using a specified method and parameters.
|
|
240
|
+
|
|
241
|
+
Parameters
|
|
242
|
+
----------
|
|
243
|
+
sample_id_key : str, optional
|
|
244
|
+
Column name in `adata.obs` for sample identifiers. Default is 'sample_id'.
|
|
245
|
+
method : str, optional
|
|
246
|
+
The method used for network inference. Options: 'maximum_likelihood', 'mcmc_maximum_likelihood', 'pseudo_likelihood', 'auto'. Default is 'auto'.
|
|
247
|
+
directed : bool, optional
|
|
248
|
+
Whether to infer a directed network. Default is False.
|
|
249
|
+
params : dict, optional
|
|
250
|
+
Additional parameters for network inference. Default is None.
|
|
251
|
+
example_list : List[str], optional
|
|
252
|
+
List of example sample identifiers. Default is None.
|
|
253
|
+
run_with_matlab : bool, optional
|
|
254
|
+
If True, prepares data for MATLAB execution instead of running inference in Python. Default is False.
|
|
259
255
|
"""
|
|
260
256
|
|
|
261
|
-
self.
|
|
257
|
+
self.sample_id_key = sample_id_key
|
|
262
258
|
|
|
263
259
|
if method not in [
|
|
264
260
|
'maximum_likelihood',
|
|
@@ -269,7 +265,7 @@ class AbstractDSPIN(ABC):
|
|
|
269
265
|
"Method must be one of 'maximum_likelihood', 'mcmc_maximum_likelihood', 'pseudo_likelihood' or 'auto'.")
|
|
270
266
|
|
|
271
267
|
if method == 'auto':
|
|
272
|
-
samp_list = np.unique(self.adata.obs[
|
|
268
|
+
samp_list = np.unique(self.adata.obs[sample_id_key])
|
|
273
269
|
num_sample = len(samp_list)
|
|
274
270
|
if example_list is not None:
|
|
275
271
|
num_sample = len(example_list)
|
|
@@ -287,18 +283,14 @@ class AbstractDSPIN(ABC):
|
|
|
287
283
|
|
|
288
284
|
print("Using {} for network inference.".format(method))
|
|
289
285
|
|
|
290
|
-
if precomputed_discretization is not None:
|
|
291
|
-
self._onmf_rep_tri = precomputed_discretization
|
|
292
|
-
|
|
293
286
|
if method == 'pseudo_likelihood':
|
|
294
|
-
self.raw_data_state(
|
|
287
|
+
self.raw_data_state(sample_id_key)
|
|
295
288
|
else:
|
|
296
|
-
self.raw_data_corr(
|
|
289
|
+
self.raw_data_corr(sample_id_key)
|
|
297
290
|
|
|
298
291
|
self.example_list = example_list
|
|
299
292
|
|
|
300
293
|
train_dat = self.default_params(method)
|
|
301
|
-
train_dat['rec_gap'] = record_step
|
|
302
294
|
train_dat['directed'] = directed
|
|
303
295
|
if params is not None:
|
|
304
296
|
train_dat.update(params)
|
|
@@ -308,7 +300,7 @@ class AbstractDSPIN(ABC):
|
|
|
308
300
|
file_path = self.save_path + 'raw_data.mat'
|
|
309
301
|
savemat(file_path, {'raw_data': self._raw_data, 'sample_list': self._samp_list,
|
|
310
302
|
'method': method, 'directed': directed, **train_dat})
|
|
311
|
-
print("Data saved to {}. Please run the network inference in MATLAB
|
|
303
|
+
print("Data saved to {}. Please run the network inference in MATLAB".format(file_path))
|
|
312
304
|
|
|
313
305
|
else:
|
|
314
306
|
cur_j, cur_h, train_log = learn_network_adam(self.raw_data, method, train_dat)
|
|
@@ -317,80 +309,131 @@ class AbstractDSPIN(ABC):
|
|
|
317
309
|
self.train_log = train_log
|
|
318
310
|
|
|
319
311
|
def response_relative_to_control(self,
|
|
320
|
-
|
|
321
|
-
|
|
312
|
+
sample_id_key: str = 'sample_id',
|
|
313
|
+
if_control_key: str = 'if_control',
|
|
314
|
+
batch_key: str = 'batch'):
|
|
322
315
|
"""
|
|
323
|
-
Compute the relative responses based on
|
|
324
|
-
|
|
325
|
-
Parameters
|
|
326
|
-
|
|
327
|
-
|
|
316
|
+
Compute the relative responses based on control samples in each batch.
|
|
317
|
+
|
|
318
|
+
Parameters
|
|
319
|
+
----------
|
|
320
|
+
sample_id_key : str, optional
|
|
321
|
+
Column name in `adata.obs` specifying sample identifiers. Default is 'sample_id'.
|
|
322
|
+
if_control_key : str, optional
|
|
323
|
+
Column name in `adata.obs` indicating whether a sample is a control. Default is 'if_control'.
|
|
324
|
+
batch_key : str, optional
|
|
325
|
+
Column name in `adata.obs` specifying batch assignment for each sample. Default is 'batch'.
|
|
328
326
|
"""
|
|
329
327
|
|
|
328
|
+
if if_control_key not in self.adata.obs.columns:
|
|
329
|
+
raise ValueError(f"Column '{if_control_key}' not found in adata.obs.")
|
|
330
|
+
if batch_key not in self.adata.obs.columns:
|
|
331
|
+
raise ValueError(f"Column '{batch_key}' not found in adata.obs.")
|
|
332
|
+
|
|
333
|
+
sample_list = self._samp_list
|
|
334
|
+
sample_to_control = self.adata.obs.groupby(sample_id_key)[if_control_key].first().to_dict()
|
|
335
|
+
sample_to_batch = self.adata.obs.groupby(sample_id_key)[batch_key].first().to_dict()
|
|
336
|
+
|
|
337
|
+
if_control = np.array([sample_to_control[sample] for sample in sample_list])
|
|
338
|
+
batch_index = np.array([sample_to_batch[sample] for sample in sample_list])
|
|
339
|
+
|
|
330
340
|
self._relative_responses = compute_relative_responses(self.responses, if_control, batch_index)
|
|
331
341
|
|
|
332
342
|
|
|
333
343
|
class GeneDSPIN(AbstractDSPIN):
|
|
334
344
|
"""
|
|
335
|
-
GeneDSPIN inherits the AbstractDSPIN class, optimized for
|
|
336
|
-
It doesn't need
|
|
345
|
+
GeneDSPIN inherits the AbstractDSPIN class, optimized for building gene-level networks.
|
|
346
|
+
It doesn't need oNMF and can directly perform network inference on the genes.
|
|
337
347
|
"""
|
|
338
348
|
|
|
339
349
|
def __init__(self,
|
|
340
350
|
adata: ad.AnnData,
|
|
341
351
|
save_path: str,
|
|
342
|
-
num_spin: int
|
|
352
|
+
num_spin: int,
|
|
353
|
+
clip_percentile: float = 95):
|
|
343
354
|
"""
|
|
344
355
|
Initialize the GeneDSPIN object.
|
|
345
356
|
|
|
346
|
-
Parameters
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
357
|
+
Parameters
|
|
358
|
+
----------
|
|
359
|
+
adata : ad.AnnData
|
|
360
|
+
Annotated single-cell gene expression data.
|
|
361
|
+
save_path : str
|
|
362
|
+
Directory path where results will be saved.
|
|
363
|
+
num_spin : int
|
|
364
|
+
Number of spins used in inference.
|
|
365
|
+
clip_percentile : float, optional
|
|
366
|
+
Percentile threshold for clipping values before discretization. Default is 95.
|
|
350
367
|
"""
|
|
351
368
|
|
|
352
369
|
super().__init__(adata, save_path, num_spin)
|
|
353
370
|
print("GeneDSPIN initialized.")
|
|
354
371
|
|
|
355
372
|
if issparse(adata.X):
|
|
356
|
-
self._onmf_rep_ori =
|
|
373
|
+
self._onmf_rep_ori = adata.X.toarray()
|
|
357
374
|
else:
|
|
358
375
|
self._onmf_rep_ori = adata.X
|
|
359
376
|
|
|
360
|
-
self.discretize()
|
|
377
|
+
self.discretize(clip_percentile)
|
|
378
|
+
|
|
379
|
+
def program_regulator_discovery(self,
|
|
380
|
+
program_representation_raw: np.ndarray,
|
|
381
|
+
sample_id_key: str = 'sample_id',
|
|
382
|
+
params: dict = None):
|
|
383
|
+
"""
|
|
384
|
+
Discover regulators of given gene programs using regression.
|
|
385
|
+
|
|
386
|
+
Parameters
|
|
387
|
+
----------
|
|
388
|
+
program_representation_raw : np.ndarray
|
|
389
|
+
The gene program representation.
|
|
390
|
+
sample_id_key : str, optional
|
|
391
|
+
Column name in `adata.obs` specifying sample identifiers. Default is 'sample_id'.
|
|
392
|
+
params : dict, optional
|
|
393
|
+
Additional parameters for regression. Default is None.
|
|
394
|
+
"""
|
|
395
|
+
|
|
396
|
+
program_states, samp_list = sample_states(self.adata.obs[sample_id_key], program_representation_raw)
|
|
397
|
+
|
|
398
|
+
gene_states, _ = sample_states(self.adata.obs[sample_id_key], self._onmf_rep_tri)
|
|
399
|
+
|
|
400
|
+
train_dat = {
|
|
401
|
+
'num_epoch': 500,
|
|
402
|
+
'stepsz': 0.01,
|
|
403
|
+
'lambda_l1_interaction': 0.01,
|
|
404
|
+
'rec_gap': 10
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
if params is not None:
|
|
408
|
+
train_dat.update(params)
|
|
409
|
+
|
|
410
|
+
cur_interaction, cur_selfj, cur_selfh = learn_program_regulators(gene_states, program_states, train_dat)
|
|
411
|
+
|
|
412
|
+
self.program_interactions = cur_interaction
|
|
413
|
+
self.program_activities = cur_selfh
|
|
414
|
+
self.program_self_interactions = cur_selfj
|
|
415
|
+
|
|
361
416
|
|
|
362
417
|
|
|
363
418
|
class ProgramDSPIN(AbstractDSPIN):
|
|
364
419
|
"""
|
|
365
|
-
ProgramDSPIN
|
|
366
|
-
|
|
420
|
+
ProgramDSPIN extends AbstractDSPIN to handle large datasets efficiently.
|
|
421
|
+
This class includes methods for subsampling, balancing gene matrices, and running repeated oNMF decompositions.
|
|
367
422
|
|
|
368
|
-
...
|
|
369
423
|
Attributes
|
|
370
424
|
----------
|
|
371
425
|
adata : ad.AnnData
|
|
372
|
-
Annotated data.
|
|
426
|
+
Annotated single-cell gene expression data.
|
|
373
427
|
save_path : str
|
|
374
|
-
|
|
428
|
+
Directory path where results will be stored.
|
|
375
429
|
num_spin : int
|
|
376
|
-
Number of spins.
|
|
377
|
-
num_onmf_components : int
|
|
378
|
-
Number of
|
|
379
|
-
|
|
380
|
-
List of
|
|
381
|
-
num_repeat : int
|
|
382
|
-
Number of
|
|
383
|
-
|
|
384
|
-
Methods
|
|
385
|
-
-------
|
|
386
|
-
subsample_matrix_balance(total_sample_size, std_clip_percentile)
|
|
387
|
-
Subsample and balance a gene matrix to achieve the desired sample size and standard deviation clipping percentile.
|
|
388
|
-
compute_onmf_repeats(num_onmf_components, num_repeat, num_subsample, seed, std_clip_percentile)
|
|
389
|
-
Compute ONMF decompositions repeatedly after subsampling of the gene matrix for each repetition.
|
|
390
|
-
summarize_onmf_result(num_onmf_components, num_repeat, summary_method)
|
|
391
|
-
Summarize the results of the repeated ONMF decompositions, integrating the components obtained from each ONMF decomposition and preprograms if provided.
|
|
392
|
-
gene_program_discovery(num_onmf_components, num_subsample, num_subsample_large, num_repeat, balance_obs, balance_method, max_sample_rate, prior_programs, summary_method)
|
|
393
|
-
Discovers gene programs by performing ONMF decomposition on the given annotated data object.
|
|
430
|
+
Number of spins used in modeling.
|
|
431
|
+
num_onmf_components : int, optional
|
|
432
|
+
Number of oNMF components.
|
|
433
|
+
prior_programs : List[List[str]], optional
|
|
434
|
+
List of predefined gene programs.
|
|
435
|
+
num_repeat : int, optional
|
|
436
|
+
Number of oNMF repetitions for stability.
|
|
394
437
|
"""
|
|
395
438
|
|
|
396
439
|
def __init__(self,
|
|
@@ -398,18 +441,25 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
398
441
|
save_path: str,
|
|
399
442
|
num_spin: int,
|
|
400
443
|
num_onmf_components: int = None,
|
|
401
|
-
|
|
444
|
+
prior_programs: List[List[str]] = None,
|
|
402
445
|
num_repeat: int = 10):
|
|
403
446
|
"""
|
|
404
447
|
Initialize the ProgramDSPIN object.
|
|
405
448
|
|
|
406
|
-
Parameters
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
449
|
+
Parameters
|
|
450
|
+
----------
|
|
451
|
+
adata : ad.AnnData
|
|
452
|
+
Annotated data matrix containing observations and gene expressions.
|
|
453
|
+
save_path : str
|
|
454
|
+
Directory path for saving results.
|
|
455
|
+
num_spin : int
|
|
456
|
+
Number of spins used in DSPIN.
|
|
457
|
+
num_onmf_components : int, optional
|
|
458
|
+
Number of oNMF components, default is None.
|
|
459
|
+
prior_programs : List[List[str]], optional
|
|
460
|
+
List of predefined gene programs.
|
|
461
|
+
num_repeat : int, optional
|
|
462
|
+
Number of repetitions for oNMF computation. Default is 10.
|
|
413
463
|
"""
|
|
414
464
|
super().__init__(adata, save_path, num_spin)
|
|
415
465
|
|
|
@@ -419,8 +469,8 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
419
469
|
self._gene_matrix_large = None
|
|
420
470
|
self._use_data_list = None
|
|
421
471
|
self.gene_program_csv = None
|
|
422
|
-
self.
|
|
423
|
-
self.preprogram_num = len(
|
|
472
|
+
self.prior_programs = None
|
|
473
|
+
self.preprogram_num = len(prior_programs) if prior_programs else 0
|
|
424
474
|
|
|
425
475
|
@property
|
|
426
476
|
def onmf_decomposition(self):
|
|
@@ -432,20 +482,21 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
432
482
|
|
|
433
483
|
def subsample_matrix_balance(self,
|
|
434
484
|
total_sample_size: int,
|
|
435
|
-
std_clip_percentile: float = 20) ->
|
|
436
|
-
np.ndarray):
|
|
485
|
+
std_clip_percentile: float = 20) -> Tuple[np.ndarray, np.ndarray]:
|
|
437
486
|
"""
|
|
438
|
-
Subsample and balance
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
total_sample_size
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
487
|
+
Subsample and balance the gene matrix based on desired sample size and standard deviation clipping.
|
|
488
|
+
|
|
489
|
+
Parameters
|
|
490
|
+
----------
|
|
491
|
+
total_sample_size : int
|
|
492
|
+
The target number of cells after subsampling.
|
|
493
|
+
std_clip_percentile : float, optional
|
|
494
|
+
Percentile for standard deviation clipping, default is 20.
|
|
495
|
+
|
|
496
|
+
Returns
|
|
497
|
+
-------
|
|
498
|
+
Tuple[np.ndarray, np.ndarray]
|
|
499
|
+
Clipped standard deviations and the balanced, normalized gene matrix.
|
|
449
500
|
"""
|
|
450
501
|
gene_matrix = self.adata.X
|
|
451
502
|
cadata = self.adata
|
|
@@ -476,17 +527,14 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
476
527
|
strart_ind = np.sum(sampling_number[:ii])
|
|
477
528
|
end_ind = strart_ind + cur_num
|
|
478
529
|
if issparse(cadata.X):
|
|
479
|
-
gene_matrix_balanced[strart_ind: end_ind,
|
|
480
|
-
:] = cadata.X[cur_filt,
|
|
481
|
-
:][sele_ind,
|
|
482
|
-
:].toarray()
|
|
530
|
+
gene_matrix_balanced[strart_ind: end_ind, :] = cadata.X[cur_filt, :][sele_ind, :].toarray()
|
|
483
531
|
else:
|
|
484
|
-
gene_matrix_balanced[strart_ind: end_ind,
|
|
485
|
-
:] = cadata.X[cur_filt, :][sele_ind, :]
|
|
532
|
+
gene_matrix_balanced[strart_ind: end_ind, :] = cadata.X[cur_filt, :][sele_ind, :]
|
|
486
533
|
|
|
487
534
|
# Normalize the matrix by standard deviation
|
|
488
535
|
if issparse(gene_matrix_balanced):
|
|
489
536
|
gene_matrix_balanced = np.asarray(gene_matrix_balanced.toarray())
|
|
537
|
+
|
|
490
538
|
std = gene_matrix_balanced.std(axis=0)
|
|
491
539
|
std_clipped = std.clip(np.percentile(std, std_clip_percentile), np.inf)
|
|
492
540
|
gene_matrix_balanced_normalized = gene_matrix_balanced / std_clipped
|
|
@@ -498,42 +546,48 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
498
546
|
num_repeat: int,
|
|
499
547
|
num_subsample: int,
|
|
500
548
|
seed: int = 0,
|
|
501
|
-
std_clip_percentile: float = 20):
|
|
549
|
+
std_clip_percentile: float = 20) -> None:
|
|
502
550
|
"""
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
Parameters
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
551
|
+
Perform multiple oNMF decompositions on subsampled data to enhance stability.
|
|
552
|
+
|
|
553
|
+
Parameters
|
|
554
|
+
----------
|
|
555
|
+
num_onmf_components : int
|
|
556
|
+
Number of oNMF components.
|
|
557
|
+
num_repeat : int
|
|
558
|
+
Number of repetitions for oNMF computation.
|
|
559
|
+
num_subsample : int
|
|
560
|
+
Number of cells to use while computing the oNMF
|
|
561
|
+
seed : int, optional
|
|
562
|
+
Seed for reproducibility. Default is 0.
|
|
563
|
+
std_clip_percentile : float, optional
|
|
564
|
+
Percentile for standard deviation clipping. Default is 20.
|
|
511
565
|
"""
|
|
512
566
|
|
|
513
|
-
|
|
567
|
+
prior_programs = self.prior_programs
|
|
514
568
|
adata = self.adata
|
|
515
569
|
|
|
516
|
-
# Create the directory for saving
|
|
570
|
+
# Create the directory for saving oNMF decompositions if it doesn't
|
|
517
571
|
# exist
|
|
518
572
|
os.makedirs(self.save_path + 'onmf/', exist_ok=True)
|
|
519
573
|
|
|
520
|
-
print("Computing
|
|
574
|
+
print("Computing oNMF decomposition...")
|
|
521
575
|
|
|
522
|
-
# Repeat
|
|
576
|
+
# Repeat oNMF decomposition for num_repeat times
|
|
523
577
|
for ii in range(num_repeat):
|
|
524
578
|
np.random.seed(seed + ii)
|
|
525
|
-
# Check if
|
|
579
|
+
# Check if oNMF decomposition for the current iteration exists
|
|
526
580
|
if os.path.exists(
|
|
527
581
|
self.save_path + 'onmf/onmf_rep_{}_{}.npy'.format(num_onmf_components, ii)):
|
|
528
582
|
print(
|
|
529
|
-
"
|
|
583
|
+
"oNMF decomposition {} already exists. Skipping...".format(ii))
|
|
530
584
|
continue
|
|
531
585
|
|
|
532
586
|
# Subsample the matrix and normalize it
|
|
533
587
|
_, gene_matrix_norm = self.subsample_matrix_balance(
|
|
534
588
|
num_subsample, std_clip_percentile=std_clip_percentile)
|
|
535
589
|
|
|
536
|
-
# Compute current
|
|
590
|
+
# Compute current oNMF decomposition and save the result
|
|
537
591
|
current_onmf = compute_onmf(
|
|
538
592
|
seed + ii, num_onmf_components, gene_matrix_norm[:, self.prior_programs_mask])
|
|
539
593
|
np.save(self.save_path +
|
|
@@ -542,18 +596,24 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
542
596
|
def summarize_onmf_result(self,
|
|
543
597
|
num_onmf_components: int,
|
|
544
598
|
num_repeat: int,
|
|
545
|
-
summary_method: str):
|
|
599
|
+
summary_method: str) -> NMF:
|
|
546
600
|
"""
|
|
547
|
-
Summarize the results of
|
|
548
|
-
integrating
|
|
549
|
-
|
|
550
|
-
Parameters
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
601
|
+
Summarize the results of repeated oNMF decompositions,
|
|
602
|
+
integrating components from each oNMF run and prior programs if available.
|
|
603
|
+
|
|
604
|
+
Parameters
|
|
605
|
+
----------
|
|
606
|
+
num_onmf_components : int
|
|
607
|
+
Number of oNMF components.
|
|
608
|
+
num_repeat : int
|
|
609
|
+
Number of oNMF repetitions.
|
|
610
|
+
summary_method : str
|
|
611
|
+
The method used for summarizing oNMF components.
|
|
612
|
+
|
|
613
|
+
Returns
|
|
614
|
+
-------
|
|
615
|
+
NMF
|
|
616
|
+
An NMF object containing the summarized components.
|
|
557
617
|
"""
|
|
558
618
|
|
|
559
619
|
# Retrieve the number of spins and annotated data
|
|
@@ -567,14 +627,11 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
567
627
|
for ii in range(num_repeat):
|
|
568
628
|
cur_onmf = np.load(
|
|
569
629
|
self.save_path +
|
|
570
|
-
'onmf/onmf_rep_{}_{}.npy'.format(
|
|
571
|
-
num_onmf_components,
|
|
572
|
-
ii),
|
|
630
|
+
'onmf/onmf_rep_{}_{}.npy'.format(num_onmf_components, ii),
|
|
573
631
|
allow_pickle=True).item()
|
|
574
632
|
rec_components[ii] = cur_onmf.components_
|
|
575
633
|
|
|
576
|
-
all_components = rec_components.reshape(-1,
|
|
577
|
-
np.sum(prior_programs_mask))
|
|
634
|
+
all_components = rec_components.reshape(-1, np.sum(prior_programs_mask))
|
|
578
635
|
|
|
579
636
|
gene_group_ind = summary_components(
|
|
580
637
|
all_components,
|
|
@@ -620,23 +677,30 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
620
677
|
prior_programs: List[List[str]] = None,
|
|
621
678
|
summary_method: str = 'kmeans'):
|
|
622
679
|
"""
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
Parameters
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
680
|
+
Discover gene programs by performing oNMF decomposition on the annotated data.
|
|
681
|
+
|
|
682
|
+
Parameters
|
|
683
|
+
----------
|
|
684
|
+
num_onmf_components : int, optional
|
|
685
|
+
Number of oNMF components. Default is None.
|
|
686
|
+
num_subsample : int, optional
|
|
687
|
+
Number of samples to use after subsampling. Default is 10000.
|
|
688
|
+
num_subsample_large : int, optional
|
|
689
|
+
Number of samples for large subsampling. Default is None.
|
|
690
|
+
num_repeat : int, optional
|
|
691
|
+
Number of times to repeat oNMF. Default is 10.
|
|
692
|
+
seed : int, optional
|
|
693
|
+
Random seed for reproducibility. Default is 0.
|
|
694
|
+
balance_obs : str, optional
|
|
695
|
+
Observation category to balance. Default is None.
|
|
696
|
+
balance_method : str, optional
|
|
697
|
+
Method used for balancing categories. Default is None.
|
|
698
|
+
max_sample_rate : float, optional
|
|
699
|
+
Maximum sampling rate. Default is 2.
|
|
700
|
+
prior_programs : List[List[str]], optional
|
|
701
|
+
List of predefined gene programs. Default is None.
|
|
702
|
+
summary_method : str, optional
|
|
703
|
+
Method to summarize oNMF components. Default is 'kmeans'.
|
|
640
704
|
"""
|
|
641
705
|
|
|
642
706
|
if balance_method not in ['equal', 'proportional', 'squareroot', None]:
|
|
@@ -671,7 +735,7 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
671
735
|
gene for program in prior_programs for gene in program]
|
|
672
736
|
if len(prior_programs) > num_spin:
|
|
673
737
|
raise ValueError(
|
|
674
|
-
'Number of
|
|
738
|
+
'Number of prior_programs must be less than the number of spins')
|
|
675
739
|
|
|
676
740
|
# Generate mask to exclude genes in prior programs from the
|
|
677
741
|
# annotated data
|
|
@@ -680,7 +744,7 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
680
744
|
prior_program_mask = np.ones(adata.shape[1], dtype=bool)
|
|
681
745
|
prior_program_ind = []
|
|
682
746
|
|
|
683
|
-
# number of
|
|
747
|
+
# number of oNMF components = number of spins - number of prior
|
|
684
748
|
# programs
|
|
685
749
|
if num_onmf_components is None:
|
|
686
750
|
num_onmf_components = self.num_spin - len(prior_program_ind)
|
|
@@ -693,7 +757,7 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
693
757
|
self.matrix_std, self.large_subsample_matrix = self.subsample_matrix_balance(
|
|
694
758
|
num_subsample_large, std_clip_percentile=20)
|
|
695
759
|
|
|
696
|
-
# Compute
|
|
760
|
+
# Compute oNMF decompositions repeatedly
|
|
697
761
|
self.compute_onmf_repeats(
|
|
698
762
|
num_onmf_components,
|
|
699
763
|
num_repeat,
|
|
@@ -701,7 +765,7 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
701
765
|
seed,
|
|
702
766
|
std_clip_percentile=20)
|
|
703
767
|
|
|
704
|
-
# Summarize the
|
|
768
|
+
# Summarize the oNMF decompositions
|
|
705
769
|
onmf_summary = self.summarize_onmf_result(
|
|
706
770
|
num_onmf_components, num_repeat, summary_method)
|
|
707
771
|
self._onmf_summary = onmf_summary
|
|
@@ -722,7 +786,7 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
722
786
|
gene_matrix = self.adata.X
|
|
723
787
|
if issparse(gene_matrix):
|
|
724
788
|
gene_matrix = np.asarray(gene_matrix.toarray()).astype(np.float64)
|
|
725
|
-
# Transform the original matrix by the
|
|
789
|
+
# Transform the original matrix by the oNMF summary components and
|
|
726
790
|
# normalize by standard deviation
|
|
727
791
|
self._onmf_rep_ori = onmf_summary.transform(
|
|
728
792
|
gene_matrix / self.matrix_std)
|
|
@@ -732,11 +796,8 @@ class ProgramDSPIN(AbstractDSPIN):
|
|
|
732
796
|
|
|
733
797
|
class DSPIN(object):
|
|
734
798
|
"""
|
|
735
|
-
DSPIN class
|
|
736
|
-
|
|
737
|
-
This class serves as a automatic conditional constructor to decide
|
|
738
|
-
which subclass of DSPIN to instantiate based on the number of genes
|
|
739
|
-
and spins provided.
|
|
799
|
+
DSPIN class determines the appropriate subclass (GeneDSPIN or ProgramDSPIN)
|
|
800
|
+
based on the dataset's gene count and spin count.
|
|
740
801
|
"""
|
|
741
802
|
|
|
742
803
|
def __new__(cls,
|
|
@@ -744,32 +805,40 @@ class DSPIN(object):
|
|
|
744
805
|
save_path: str,
|
|
745
806
|
num_spin: int = None,
|
|
746
807
|
filter_threshold: float = 0.02,
|
|
747
|
-
**kwargs):
|
|
808
|
+
**kwargs) -> Union[GeneDSPIN, ProgramDSPIN]:
|
|
748
809
|
"""
|
|
749
|
-
Initialize the
|
|
750
|
-
|
|
751
|
-
Parameters
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
810
|
+
Initialize DSPIN and select the appropriate subclass.
|
|
811
|
+
|
|
812
|
+
Parameters
|
|
813
|
+
----------
|
|
814
|
+
adata : ad.AnnData
|
|
815
|
+
The annotated data matrix (n_cells x n_genes).
|
|
816
|
+
save_path : str
|
|
817
|
+
Path to save results.
|
|
818
|
+
num_spin : int, optional
|
|
819
|
+
Number of spins for network inference. Default is 15 or number of genes if lower.
|
|
820
|
+
filter_threshold : float, optional
|
|
821
|
+
Minimum fraction of cells expressing a gene for it to be retained. Default is 0.02.
|
|
822
|
+
**kwargs : dict
|
|
823
|
+
Additional keyword arguments passed to the selected subclass.
|
|
824
|
+
|
|
825
|
+
Returns
|
|
826
|
+
-------
|
|
827
|
+
Union[GeneDSPIN, ProgramDSPIN]
|
|
828
|
+
An instance of either GeneDSPIN or ProgramDSPIN based on gene count.
|
|
760
829
|
"""
|
|
761
830
|
|
|
831
|
+
if adata.X.min() < 0:
|
|
832
|
+
warnings.warn("Negative expression detected. Expect normalized log-transformed data.")
|
|
833
|
+
|
|
762
834
|
# Filtering genes based on the filter_threshold.
|
|
763
|
-
# It retains only those genes that have expression
|
|
764
|
-
# in more than the specified percentage of cells.
|
|
765
835
|
sc.pp.filter_genes(adata, min_cells=adata.shape[0] * filter_threshold)
|
|
766
836
|
print(
|
|
767
837
|
'{} genes have expression in more than {} of the cells'.format(
|
|
768
838
|
adata.shape[1],
|
|
769
839
|
filter_threshold))
|
|
770
840
|
|
|
771
|
-
# if the number of genes is less than the number of spins, use
|
|
772
|
-
# GeneDSPIN
|
|
841
|
+
# if the number of genes is less than the number of spins, use GeneDSPIN
|
|
773
842
|
if num_spin is None:
|
|
774
843
|
num_spin = np.min([15, adata.shape[1]])
|
|
775
844
|
|
|
@@ -21,7 +21,7 @@ install_requires = [
|
|
|
21
21
|
# ]
|
|
22
22
|
|
|
23
23
|
# Managing your version (adjust accordingly)
|
|
24
|
-
version = "1.
|
|
24
|
+
version = "1.2.0" # You can manage versioning in a more sophisticated way if needed
|
|
25
25
|
|
|
26
26
|
# Long description (usually a README file)
|
|
27
27
|
readme = open("README_PyPI.md").read() # Adjust the file name if necessary
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|