dspin 1.0.7__tar.gz → 1.0.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dspin
3
- Version: 1.0.7
3
+ Version: 1.0.8
4
4
  Summary: Regulatory network models from single-cell perturbation profiling
5
5
  Home-page: https://github.com/JialongJiang/DSPIN
6
6
  Author: Jialong Jiang
@@ -0,0 +1,421 @@
1
+ # -*-coding:utf-8 -*-
2
+ '''
3
+ @Time : 2024/04/02 14:29
4
+ @Author : Jialong Jiang
5
+ '''
6
+
7
+
8
+ import csv
9
+ import pandas as pd
10
+ from typing import List
11
+ import numpy as np
12
+ import os
13
+ import logging
14
+
15
+
16
+ def automatic_gpt_annotation(data_folder, file_name, gpt_model="gpt-4-0125-preview", temp=0.0, max_gene=30):
17
+ """
18
+ Main function to automate GPT annotation for a set of genes.
19
+ """
20
+ # Validate input file extension
21
+ assert file_name.endswith('.csv'), 'Input file must be a CSV file'
22
+
23
+ try:
24
+ import openai
25
+ except ImportError:
26
+ raise ImportError('Please install the required packages: openai')
27
+
28
+ # Setup and read data
29
+ gene_df, all_gene = read_data(data_folder, file_name)
30
+ num_list = gene_df.shape[1]
31
+
32
+ output_dir = data_folder + file_name.replace('.csv', '_gpt/')
33
+ context = "You are an efficient and insightful assistant to a molecular biologist"
34
+
35
+ os.makedirs(output_dir, exist_ok=True)
36
+
37
+ for ii in range(num_list):
38
+
39
+ cur_gene_list = gene_df[str(ii)]
40
+ cur_gene_list = cur_gene_list[~cur_gene_list.isna()][: max_gene]
41
+ cur_prompt = make_user_prompt_with_score(cur_gene_list)
42
+
43
+ response, fingerprint = openai_chat(context, cur_prompt, gpt_model, temp)
44
+
45
+ output_file = output_dir + f'program_{ii}.txt'
46
+
47
+ with open(output_file, 'w') as f:
48
+ f.write(response)
49
+
50
+
51
+ def automatic_david_annotation(data_folder, file_name, token_name, species='human'):
52
+ """
53
+ Main function to automate DAVID annotation for a set of genes.
54
+ """
55
+ # Validate input file extension
56
+ assert file_name.endswith('.csv'), 'Input file must be a CSV file'
57
+
58
+ try:
59
+ import mygene
60
+ from suds.client import Client
61
+ import xlsxwriter
62
+ except ImportError:
63
+ raise ImportError('Please install the required packages: mygene, suds, xlsxwriter')
64
+
65
+ # Setup and read data
66
+ all_onmf_df, all_gene = read_data(data_folder, file_name)
67
+ num_list = all_onmf_df.shape[1]
68
+
69
+ # Gene ID conversion
70
+ gene_id_map = convert_gene_ids(all_gene, species)
71
+
72
+ # Setup DAVID Web Service client
73
+ client = setup_david_client(token_name)
74
+
75
+ # Process each gene list
76
+ process_gene_lists(data_folder, file_name, all_onmf_df, gene_id_map, client)
77
+
78
+ gene_id_map_reverse = {v: k for k, v in gene_id_map.items()}
79
+
80
+ # Save results to Excel
81
+ save_to_excel(data_folder, file_name, gene_id_map_reverse, num_list, full=False)
82
+ save_to_excel(data_folder, file_name, gene_id_map_reverse, num_list, full=True)
83
+
84
+
85
+ def make_user_prompt_with_score(genes, feature_df = [], direct = False, customized_prompt = None):
86
+ """
87
+ This function is adapted from "Evaluation of large language models for discovery of gene set function"
88
+ Arxiv: arxiv.org/abs/2309.04019v2
89
+ GitHub: github.com/idekerlab/llm_evaluation_for_gene_set_interpretation
90
+
91
+ Create a "one shot" ChatGPT prompt based on the list of genes.
92
+ :return: A string containing the ChatGPT prompt text
93
+ """
94
+
95
+ general_analysis_instructions = """
96
+ Be concise, do not use unnecessary words.
97
+ Be factual, do not editorialize.
98
+ Be specific, avoid overly general statements such as 'the proteins are involved in various cellular processes'.
99
+ Avoid listing facts about individual proteins. Instead, try to group proteins with similar functions and discuss their interplay, synergistyc or antagonistic effects and functional integration within the system.
100
+ Also avoid choosing generic process names such as 'Cellular Signaling and Regulation'.
101
+ If you cannot identify a prominent biological process for the proteins in the system, I want you to communicate this in you analysis and name the process: “System of unrelated proteins”. Provide a score of 0.00 for a "System of unrelated proteins".
102
+ """
103
+
104
+ task_instructions = """
105
+ Write a critical analysis of the biological processes performed by this system of interacting proteins.
106
+ Base your analysis on prior knowledge available in your training data.
107
+ After completing your analysis, propose a brief and detailed name for the most prominent biological process performed by the system.
108
+ """
109
+
110
+ score_instructions = """
111
+ After completing your analysis, please also assign a confidence score to the process name you selected.
112
+ This score should follow the name in parentheses and range from 0.00 to 1.00. A score of 0.00 indicates the lowest confidence,
113
+ while 1.00 reflects the highest confidence. This score helps gauge how accurately the chosen name represents the functions and activities
114
+ within the system of interacting proteins. When determining your score, consider the proportion of genes in the protein system that participate
115
+ in the identified biological process. For instance, if you select "Ribosome biogenesis" as the process name but only a few genes in the system
116
+ contribute to this process, the score should be lower compared to a scenario where a majority of the genes are involved in "Ribosome biogenesis".
117
+ """
118
+
119
+ direct_instructions = """
120
+ Propose a name and provide analysis for the following gene set.
121
+ """
122
+
123
+ format_placeholder = """
124
+ Put your chosen name at the top of the analysis as 'Process: <name>’.
125
+ """
126
+
127
+ example_analysis = """
128
+ To help you in your work, I am providing an example system of interacting proteins and the corresponding example analysis output.
129
+
130
+ The example system of interacting proteins is:
131
+ PDX1, SLC2A2, NKX6-1, GLP1, GCG.
132
+
133
+ The example analysis output is:
134
+
135
+ Process: Pancreatic development and glucose homeostasis (0.96)
136
+
137
+ 1. PDX1 is a homeodomain transcription factor involved in the specification of the early pancreatic epithelium and its subsequent differentiation.
138
+ It activates the transcription of several genes including insulin, somatostatin, glucokinase and glucose transporter type 2.
139
+ It is essential for maintenance of the normal hormone-producing phenotype in the pancreatic beta-cell.
140
+ In pancreatic acinar cells, forms a complex with PBX1b and MEIS2b and mediates the activation of the ELA1 enhancer.
141
+
142
+ 2. NKX6-1 is also a transcription factor involved in the development of pancreatic beta-cells during the secondary transition.
143
+ Together with NKX2-2 and IRX3, controls the generation of motor neurons in the neural tube and belongs to the neural progenitor
144
+ factors induced by Sonic Hedgehog (SHH) signals.
145
+
146
+ 3.GCG and GLP1, respectively glucagon and glucagon-like peptide 1, are involved in glucose metabolism and homeostasis.
147
+ GCG raises blood glucose levels by promoting gluconeogenesis and is the counter regulatory hormone of Insulin.
148
+ GLP1 is a potent stimulator of Glucose-Induced Insulin Secretion (GSIS). Plays roles in gastric motility and suppresses blood glucagon levels.
149
+ Promotes growth of the intestinal epithelium and pancreatic islet mass both by islet neogenesis and islet cell proliferation.
150
+
151
+ 4. SLC2A2, also known as GLUT2, is a facilitative hexose transporter. In hepatocytes, it mediates bi-directional transport of glucose accross the plasma membranes,
152
+ while in the pancreatic beta-cell, it is the main transporter responsible for glucose uptake and part of the cell's glucose-sensing mechanism.
153
+ It is involved in glucose transport in the small intestine and kidney too.
154
+
155
+ To summarize, the genes in this set are involved in the specification, differentiation, growth and functionality of the pancreas,
156
+ with a particular emphasis on the pancreatic beta-cell. Particularly, the architecture of the pancreatic islet ensures proper glucose sensing
157
+ and homeostasis via a number of different hormones and receptors that can elicit both synergistic and antagonistic effects in the pancreas itself and other peripheral tissues.
158
+ """
159
+
160
+ if direct == True:
161
+ prompt_text = direct_instructions
162
+ prompt_text += format_placeholder
163
+ elif customized_prompt:
164
+ prompt_text = customized_prompt
165
+ prompt_text += format_placeholder
166
+ else:
167
+ prompt_text = task_instructions
168
+ prompt_text += score_instructions
169
+ prompt_text += format_placeholder
170
+ prompt_text += general_analysis_instructions
171
+ prompt_text += example_analysis
172
+
173
+ prompt_text += "\n\nHere are the interacting proteins:\n"
174
+ prompt_text += f'\nProteins: '
175
+ prompt_text += ", ".join(genes) + ".\n\n"
176
+
177
+ if feature_df:
178
+ prompt_text += "\n\nHere are the gene features:\n"
179
+ prompt_text = add_gene_feature_summary(prompt_text, feature_df)
180
+
181
+ return prompt_text
182
+
183
+
184
+ def openai_chat(context, prompt, model, temperature, seed: int = None):
185
+
186
+ try:
187
+ import openai
188
+ except ImportError:
189
+ raise ImportError('Please install the required packages: openai')
190
+
191
+ try:
192
+ openai.api_key = os.environ['OPENAI_API_KEY']
193
+ except:
194
+ raise Exception("API key not found. Please set the OPENAI_API_KEY environment variable.")
195
+
196
+ backoff_time = 10 # Start backoff time at 10 second
197
+ max_tokens = 2000
198
+
199
+ response = openai.chat.completions.create(
200
+ model=model,
201
+ messages=[{"role": "system", "content": context},{"role": "user", "content": prompt}],
202
+ max_tokens=max_tokens,
203
+ n=1,
204
+ stop=None,
205
+ seed=seed,
206
+ temperature=temperature,
207
+ )
208
+
209
+ # print(response)
210
+ # tokens_used = response["usage"]["total_tokens"]
211
+ tokens_used = response.usage.total_tokens
212
+ response_content = response.choices[0].message.content
213
+ system_fingerprint = response.system_fingerprint
214
+
215
+ return response_content, system_fingerprint
216
+
217
+
218
+ def read_data(data_folder, file_name):
219
+ """
220
+ Read gene data from a CSV file and return a unique list of gene names.
221
+
222
+ Parameters:
223
+ data_folder (str): The folder where the data file is located.
224
+ file_name (str): The name of the CSV file containing gene data.
225
+
226
+ Returns:
227
+ list: A unique list of gene names.
228
+ """
229
+ program_file_name = os.path.join(data_folder, file_name)
230
+ gene_df = pd.read_csv(program_file_name)
231
+ gene_list = gene_df.values.flatten().astype('str')
232
+ unique_genes = list(np.unique(gene_list))
233
+ unique_genes.remove('nan')
234
+ return gene_df, unique_genes
235
+
236
+
237
+ def convert_gene_ids(all_gene, species):
238
+ """
239
+ Convert gene symbols to ENSEMBL IDs using the MyGeneInfo service.
240
+
241
+ Parameters:
242
+ all_gene (list): A list of gene symbols.
243
+ species (str): The species for gene ID conversion (default is 'human').
244
+
245
+ Returns:
246
+ dict: A dictionary mapping gene symbols to ENSEMBL IDs.
247
+ """
248
+ mg = mygene.MyGeneInfo()
249
+ result = mg.querymany(all_gene, scopes="symbol", fields="ensembl.gene", species=species)
250
+ gene_id_map = {}
251
+ for gene_info in result:
252
+ if 'ensembl' in gene_info:
253
+ ensembl_info = gene_info['ensembl']
254
+ ensembl_id = ensembl_info['gene'] if isinstance(ensembl_info, dict) else ensembl_info[0]['gene']
255
+ gene_id_map[gene_info['query']] = ensembl_id
256
+ return gene_id_map
257
+
258
+
259
+ def setup_david_client(token_name):
260
+ """
261
+ Setup and authenticate with the DAVID Web Service.
262
+
263
+ Parameters:
264
+ token_name (str): The authentication token for DAVID Web Service.
265
+
266
+ Returns:
267
+ Client: An authenticated suds client for DAVID Web Service.
268
+ """
269
+ url = 'https://david.ncifcrf.gov/webservice/services/DAVIDWebService?wsdl'
270
+ client = Client(url)
271
+ client.wsdl.services[0].setlocation('https://david.ncifcrf.gov/webservice/services/DAVIDWebService.DAVIDWebServiceHttpSoap11Endpoint/')
272
+ client.service.authenticate(token_name)
273
+ return client
274
+
275
+
276
+ def parse_david_output(term_clustering_report, save_path):
277
+ """
278
+ Parse the output from DAVID and save to a file.
279
+
280
+ Parameters:
281
+ term_clustering_report: The term clustering report from DAVID Web Service.
282
+ save_path (str): The file path to save the parsed output.
283
+ """
284
+ total_clusters = len(term_clustering_report)
285
+ print('Total clusters:', total_clusters)
286
+ with open(save_path, 'w') as f_out:
287
+ for i, cluster_record in enumerate(term_clustering_report, start=1):
288
+ enrichment_score = cluster_record.score
289
+ f_out.write(f'Annotation Cluster {i}\tEnrichmentScore:{enrichment_score}\n')
290
+ headers = ['Category', 'Term', 'Count', '%', 'Pvalue', 'Genes', 'List Total', 'Pop Hits', 'Pop Total', 'Fold Enrichment', 'Bonferroni', 'Benjamini', 'FDR']
291
+ f_out.write('\t'.join(headers) + '\n')
292
+ for chart_record in cluster_record.simpleChartRecords:
293
+ row = [
294
+ chart_record.categoryName,
295
+ chart_record.termName,
296
+ str(chart_record.listHits),
297
+ str(chart_record.percent),
298
+ str(chart_record.ease),
299
+ chart_record.geneIds,
300
+ str(chart_record.listTotals),
301
+ str(chart_record.popHits),
302
+ str(chart_record.popTotals),
303
+ str(chart_record.foldEnrichment),
304
+ str(chart_record.bonferroni),
305
+ str(chart_record.benjamini),
306
+ str(chart_record.afdr)
307
+ ]
308
+ f_out.write('\t'.join(row) + '\n')
309
+
310
+
311
+ def process_gene_lists(data_folder, file_name, all_onmf_df, gene_id_map, client):
312
+ """
313
+ Process each gene list, communicate with DAVID, and parse the results.
314
+
315
+ Parameters:
316
+ data_folder (str): The folder where the data files are located.
317
+ file_name (str): The name of the original CSV file containing gene data.
318
+ all_onmf_df (pd.DataFrame): A DataFrame containing gene lists.
319
+ gene_id_map (dict): A dictionary mapping gene symbols to ENSEMBL IDs.
320
+ client (Client): An authenticated suds client for DAVID Web Service.
321
+ """
322
+
323
+ def map_gene_ids(gene_id_map, gene_list):
324
+ return gene_list.map(gene_id_map).fillna(gene_list)
325
+
326
+ num_lists = all_onmf_df.shape[1]
327
+ save_folder = os.path.join(data_folder, file_name.replace('.csv', '_david_results/'))
328
+ os.makedirs(save_folder, exist_ok=True)
329
+
330
+ for i in range(num_lists):
331
+ cur_gene_list = all_onmf_df.iloc[:, i].dropna()
332
+ cur_gene_list = map_gene_ids(gene_id_map, cur_gene_list)
333
+ save_path = os.path.join(save_folder, f'list_{i}.csv')
334
+
335
+ input_ids = ','.join(cur_gene_list)
336
+ id_type = 'ENSEMBL_GENE_ID'
337
+ list_name = f'List_{i}'
338
+ list_type = 0
339
+
340
+ client.service.addList(input_ids, id_type, list_name, list_type)
341
+ term_clustering_report = client.service.getTermClusterReport(3, 3, 3, 0.5, 50)
342
+ print(f'Gene list {i} ', end='\t')
343
+ parse_david_output(term_clustering_report, save_path)
344
+
345
+
346
+ def save_to_excel(data_folder, file_name, gene_id_map_reverse, num_lists, full=True):
347
+ """
348
+ Save the parsed DAVID output to an Excel file, with an option to include full or filtered results.
349
+
350
+ Parameters:
351
+ data_folder (str): The folder where the data files are located.
352
+ file_name (str): The name of the original CSV file containing gene data.
353
+ gene_id_map_reverse (dict): A dictionary mapping ENSEMBL IDs to gene symbols.
354
+ num_lists (int): The number of gene lists processed.
355
+ full (bool): Whether to save the full results. If False, only results with Pvalue <= 0.05 are saved.
356
+ """
357
+
358
+ def map_gene_id_reverse(gene_name):
359
+ try:
360
+ return gene_id_map_reverse[gene_name]
361
+ except:
362
+ return gene_name
363
+
364
+ def map_gene_id_reverse_list(gene_list):
365
+ try:
366
+ return ','.join([map_gene_id_reverse(gene) for gene in gene_list.split(', ')])
367
+ except:
368
+ return gene_list
369
+
370
+ def format_pvalue(pvalue):
371
+ try:
372
+ if isinstance(pvalue, str):
373
+ return '%.1e' % float(pvalue)
374
+ else:
375
+ return pvalue
376
+ except:
377
+ return pvalue
378
+
379
+ def format_percent(value):
380
+ try:
381
+ if isinstance(value, str):
382
+ return int(float(value))
383
+ else:
384
+ return value
385
+ except:
386
+ return value
387
+
388
+ def filt_fun(pvalue):
389
+ try:
390
+ return float(pvalue) > 0.05
391
+ except:
392
+ return False
393
+
394
+ save_folder = os.path.join(data_folder, file_name.replace('.csv', '_david_results/'))
395
+ xlsx_name = file_name.replace('.csv', '_david_full.xlsx') if full else file_name.replace('.csv', '_david.xlsx')
396
+
397
+ with pd.ExcelWriter(data_folder + xlsx_name) as writer:
398
+ for ii in range(num_lists):
399
+ cur_pd = pd.read_csv(save_folder + 'list_%d.csv' % ii, sep='\t', names=['Category', 'Term', 'Count', '%', 'Pvalue', 'Genes', 'List Total', 'Pop Hits', 'Pop Total', 'Fold Enrichment', 'Bonferroni', 'Benjamini', 'FDR'])
400
+ filt1 = cur_pd['Category'] == 'UP_SEQ_FEATURE'
401
+ filt2 = cur_pd['Pvalue'].apply(filt_fun)
402
+ if not full:
403
+ cur_pd = cur_pd[~(filt1 | filt2)]
404
+
405
+ cur_pd['Term'] = cur_pd['Term'].apply(lambda x: x.split('~')[- 1])
406
+ cur_pd['Genes'] = cur_pd['Genes'].apply(map_gene_id_reverse_list)
407
+ cur_pd['Pvalue'] = cur_pd['Pvalue'].apply(format_pvalue)
408
+ cur_pd['%'] = cur_pd['%'].apply(format_percent)
409
+
410
+ ind_insert = np.where(cur_pd['Category'].str.startswith('Annotation Cluster'))[0]
411
+ if len(ind_insert) > 1:
412
+ for kk, ind in enumerate(ind_insert[1: ]):
413
+ cur_pd = pd.DataFrame(np.insert(cur_pd.values, ind + kk, values=[" "] * len(cur_pd.columns), axis=0),columns = cur_pd.columns)
414
+
415
+ cur_pd.to_excel(writer, sheet_name='program_%d' % ii, index=False)
416
+ writer.sheets['program_%d' % ii].set_column(0, 0, 25)
417
+ writer.sheets['program_%d' % ii].set_column(1, 1, 50)
418
+ writer.sheets['program_%d' % ii].set_column(2, 2, 5)
419
+ writer.sheets['program_%d' % ii].set_column(3, 3, 5)
420
+ writer.sheets['program_%d' % ii].set_column(5, 5, 100)
421
+
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dspin
3
- Version: 1.0.7
3
+ Version: 1.0.8
4
4
  Summary: Regulatory network models from single-cell perturbation profiling
5
5
  Home-page: https://github.com/JialongJiang/DSPIN
6
6
  Author: Jialong Jiang
@@ -10,9 +10,9 @@ __pycache__/dspin_abstract.cpython-310.pyc
10
10
  __pycache__/plot.cpython-310.pyc
11
11
  __pycache__/plotting.cpython-37.pyc
12
12
  dspin/__init__.py
13
+ dspin/annotate.py
13
14
  dspin/compute.py
14
15
  dspin/dspin.py
15
- dspin/gpt.py
16
16
  dspin/plot.py
17
17
  dspin.egg-info/PKG-INFO
18
18
  dspin.egg-info/SOURCES.txt
@@ -21,7 +21,7 @@ install_requires = [
21
21
  # ]
22
22
 
23
23
  # Managing your version (adjust accordingly)
24
- version = "1.0.7" # You can manage versioning in a more sophisticated way if needed
24
+ version = "1.0.8" # You can manage versioning in a more sophisticated way if needed
25
25
 
26
26
  # Long description (usually a README file)
27
27
  readme = open("README_PyPI.md").read() # Adjust the file name if necessary
dspin-1.0.7/dspin/gpt.py DELETED
@@ -1,65 +0,0 @@
1
- import openai
2
- import csv
3
- import pandas as pd
4
- from typing import List
5
-
6
- system_question = "Your are a desperate biology phd professor studying gene expression from a dataset of cancer patients. Given following a list of genes in the gene program, you have two tasks: \
7
- Firstly, summarize functions of the program in a few important functions. \
8
- Secondly, give a higher-level name of the gene program based on its function. You are required by your mentor to name it as short as possible.\
9
- You get 100 score if you can name it in two or three words, 80 score if you can name it in four words, 20 score for five or more words. \
10
- A limitation to this general rule is that you will lose 20 score if you name a gene program similar 'Gene Expression Program' or 'Gene Regulation Progran'. \
11
- Please also avoid 'cancer' in the name you give. \
12
- The results should be in the format: name: (your answer) \n function 1: (), related genes: ();\
13
- function 2: (), related genes: ();\
14
- ..... (more functions and their annotation, please don't list more than 10 functions)\
15
- You will be given a list of genes in the same gene program in each user input. "
16
-
17
- def init_openai_api(api_key):
18
- openai.api_key = api_key
19
-
20
- def gene_program_csv_to_list(file_path) -> List[str]:
21
- data = pd.read_csv(file_path)
22
- gene_programs = []
23
- for col in data.columns:
24
- column_values = data[col].dropna().tolist()
25
- gene_programs.append(','.join(column_values))
26
- return gene_programs
27
-
28
-
29
- def generate_response(gene_program):
30
- messages = list()
31
- messages.append({"role": "system", "content": system_question})
32
- messages.append({"role": "user", "content": gene_program})
33
- try:
34
- response = openai.ChatCompletion.create(
35
- model = "gpt-3.5-turbo",
36
- messages=messages, # The starting prompt for the AI
37
- max_tokens=1024, # Adjust the desired response length
38
- temperature=0, # Controls the randomness of the response
39
- n=1, # Number of responses to generate
40
- )
41
- # Get the response text from the API response
42
- response_text = response['choices'][0]['message']['content']
43
- return response_text
44
- except Exception as e:
45
- return str(e)
46
-
47
- def generate_responses_to_csv(file_path, output_file_path):
48
- gene_programs = gene_program_csv_to_list(file_path)
49
- data = []
50
- responses = []
51
- for i in range(49,50):
52
- data_string = generate_response(gene_programs[i])
53
- responses.append(data_string)
54
- colon_index = data_string.find(':')
55
- newline_index = data_string.find('\n')
56
- name = data_string[colon_index + 1: newline_index]
57
- name = name.strip('Program')
58
- function = data_string[newline_index + 1:]
59
- data[i] = (i, name, function, gene_programs[i])
60
- print(str(i) + " program done" )
61
-
62
- with open(output_file_path, 'w', newline = '') as f:
63
- writer = csv.writer(f)
64
- writer.writerow(['index', 'name', 'function', 'genes'])
65
- writer.writerows(data)
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes