pkcm 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pkcm-0.0.1/MANIFEST.in +2 -0
- pkcm-0.0.1/PKG-INFO +9 -0
- pkcm-0.0.1/README.md +31 -0
- pkcm-0.0.1/binaries/pkcmalign +396 -0
- pkcm-0.0.1/binaries/pkcmbuild +181 -0
- pkcm-0.0.1/binaries/pkcmemit +68 -0
- pkcm-0.0.1/binaries/pkcmtd +132 -0
- pkcm-0.0.1/pkcm.egg-info/PKG-INFO +9 -0
- pkcm-0.0.1/pkcm.egg-info/SOURCES.txt +36 -0
- pkcm-0.0.1/pkcm.egg-info/dependency_links.txt +1 -0
- pkcm-0.0.1/pkcm.egg-info/requires.txt +5 -0
- pkcm-0.0.1/pkcm.egg-info/top_level.txt +3 -0
- pkcm-0.0.1/pkcm_rust_lib/Cargo.toml +19 -0
- pkcm-0.0.1/pkcm_rust_lib/src/helix.rs +1398 -0
- pkcm-0.0.1/pkcm_rust_lib/src/hmm.rs +242 -0
- pkcm-0.0.1/pkcm_rust_lib/src/lib.rs +1307 -0
- pkcm-0.0.1/pkcm_rust_lib/src/main.rs +37 -0
- pkcm-0.0.1/pkcm_rust_lib/target/debug/build/target-lexicon-135ffbfc19927775/out/host.rs +74 -0
- pkcm-0.0.1/pkcm_rust_lib/target/debug/build/target-lexicon-26a6be2fa7fe417f/out/host.rs +74 -0
- pkcm-0.0.1/pkcm_rust_lib/target/debug/build/target-lexicon-9efa26edf6fd65b7/out/host.rs +74 -0
- pkcm-0.0.1/pkcm_rust_lib/target/release/build/target-lexicon-0764a73534481d19/out/host.rs +74 -0
- pkcm-0.0.1/pkcm_rust_lib/target/release/build/target-lexicon-7780de2ce8d8551b/out/host.rs +74 -0
- pkcm-0.0.1/pkcm_rust_lib/target/release/build/target-lexicon-a75276a30db17f7c/out/host.rs +74 -0
- pkcm-0.0.1/pkcm_rust_lib/target/release/build/target-lexicon-e5aae973b460b6de/out/host.rs +74 -0
- pkcm-0.0.1/pkcm_rust_lib/target/release/build/target-lexicon-eee64eb8c5445764/out/host.rs +74 -0
- pkcm-0.0.1/pkcm_rust_lib/target/release/build/thiserror-dc643ac8ac6187ac/out/private.rs +5 -0
- pkcm-0.0.1/pkcm_rust_lib/tests/integration_test.rs +11 -0
- pkcm-0.0.1/pyproject.toml +8 -0
- pkcm-0.0.1/python/minimal_expansion.py +104 -0
- pkcm-0.0.1/python/scoring_scheme.py +435 -0
- pkcm-0.0.1/python/td_utils.py +448 -0
- pkcm-0.0.1/python/utils.py +2055 -0
- pkcm-0.0.1/setup.cfg +4 -0
- pkcm-0.0.1/setup.py +22 -0
- pkcm-0.0.1/tests/test_pkcmalign.py +32 -0
- pkcm-0.0.1/tests/test_pkcmbuild.py +20 -0
- pkcm-0.0.1/tests/test_pkcmemit.py +19 -0
- pkcm-0.0.1/tests/test_pkcmtd.py +19 -0
pkcm-0.0.1/MANIFEST.in
ADDED
pkcm-0.0.1/PKG-INFO
ADDED
pkcm-0.0.1/README.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
We suggest the creation of a python environment:
|
|
2
|
+
```
|
|
3
|
+
python3 -m env pkcm_env
|
|
4
|
+
source pkcm_env/bin/activate
|
|
5
|
+
```
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
The dependencies and package can be installed with:
|
|
10
|
+
```
|
|
11
|
+
pip install -r requirements.txt
|
|
12
|
+
pip install .
|
|
13
|
+
```
|
|
14
|
+
To run the code, one also need to install the pkcm rust library:
|
|
15
|
+
```
|
|
16
|
+
cd pkcm_rust_lib/
|
|
17
|
+
maturin develop
|
|
18
|
+
```
|
|
19
|
+
## Example
|
|
20
|
+
Building the model:
|
|
21
|
+
```
|
|
22
|
+
pkcmbuild pkcm_models/toy_model.pkcm seed_alignments/toy_model.stk_keep
|
|
23
|
+
```
|
|
24
|
+
Computing a tree decomposition for it
|
|
25
|
+
```
|
|
26
|
+
pkcmtd pkcm_models/toy_model.pkcm.helix
|
|
27
|
+
```
|
|
28
|
+
Aligning sequences to it:
|
|
29
|
+
```
|
|
30
|
+
pkcmalign pkcm_models/toy_model.pkcm pkcm_models/toy_model.pkcm.helix pkcm_models/toy_model.pkcm.helix.td unaligned_full_fasta/toy_model.fa test seed_alignments/toy_model.stk_keep
|
|
31
|
+
```
|
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
import warnings
|
|
3
|
+
import numpy as np
|
|
4
|
+
import subprocess
|
|
5
|
+
import argparse
|
|
6
|
+
from pkcm_rust_lib import rust_align_wrapper, QdbError, HmmQdbError
|
|
7
|
+
from pkcm.utils import get_sscons, constrained_helix_mapping_dict, extract_helix_state_graph, evaluate_alignment, best_possible_score_helix, read_pkcm_file, get_infernal_solution
|
|
8
|
+
from Bio import SeqIO
|
|
9
|
+
from Bio.Seq import Seq
|
|
10
|
+
from Bio.SeqRecord import SeqRecord
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
|
|
14
|
+
parser = argparse.ArgumentParser(description="align a set of sequences in a fasta file to a pseudoknotted covariance model.")
|
|
15
|
+
|
|
16
|
+
parser.add_argument('pkcm_file', type=str, help='.pkcm file to align to')
|
|
17
|
+
parser.add_argument('td_file', type=str, help='.td file, nice tree decomposition of the graph of variables.')
|
|
18
|
+
parser.add_argument('fasta_file', type=str, help='.fasta of sequences to align')
|
|
19
|
+
parser.add_argument('output_file', type=str, help='output file')
|
|
20
|
+
parser.add_argument('seed_alignment', type=str, help='.seed seed alignment used to build the pkcm')
|
|
21
|
+
parser.add_argument('--beta', type=float, default=0.00001, help='parameter for Query-Dependent Banding, between 0 and 1.\
|
|
22
|
+
Higher is faster, but more approximate.')
|
|
23
|
+
parser.add_argument('--tau', type=float, default=0.001, help='parameter for Query-Dependent Banding, between 0 and 1.\
|
|
24
|
+
Higher is faster, but more approximate.')
|
|
25
|
+
parser.add_argument('--exact', action='store_true', help="Deactivates any heuristic")
|
|
26
|
+
parser.add_argument('--noqdb', action='store_true', help="Deactivates QDB")
|
|
27
|
+
parser.add_argument('--nohmm', action='store_true', help="Deactivates HMM")
|
|
28
|
+
parser.add_argument('--dense', action='store_true', help="Uses a denser contiguous array instead of a hashmap.\
|
|
29
|
+
Faster, but may use more memory.")
|
|
30
|
+
parser.add_argument('-N', type=int, default=sys.maxsize, dest='limit_num_sequences')
|
|
31
|
+
|
|
32
|
+
args = parser.parse_args()
|
|
33
|
+
|
|
34
|
+
outfile = open(args.output_file,'w', buffering=1)
|
|
35
|
+
|
|
36
|
+
# Getting the ss_cons and rf lines to include
|
|
37
|
+
# them in the output.
|
|
38
|
+
for line in open(args.seed_alignment).readlines():
|
|
39
|
+
if line.startswith('#=GC SS_cons'):
|
|
40
|
+
ss_cons = line.split(' ')[-1].rstrip('\n')
|
|
41
|
+
if line.startswith('#=GC RF'):
|
|
42
|
+
rf_line = line.split(' ')[-1].rstrip('\n')
|
|
43
|
+
|
|
44
|
+
# Re-figuring out the mapping between the
|
|
45
|
+
# bounds of the helices and the
|
|
46
|
+
# half helices <-> variables
|
|
47
|
+
partner = {}
|
|
48
|
+
half_helices = []
|
|
49
|
+
helices = []
|
|
50
|
+
for line in open(args.pkcm_file).readlines():
|
|
51
|
+
if line.startswith("## END HELIX ARRANGEMENT DESCRIPTION"):
|
|
52
|
+
break
|
|
53
|
+
if line.startswith("Helix"):
|
|
54
|
+
name = line.split(' ')[0]
|
|
55
|
+
i = int(line.split(' ')[-4])-1
|
|
56
|
+
j = int(line.split(' ')[-3])-1
|
|
57
|
+
k = int(line.split(' ')[-2])-1
|
|
58
|
+
l = int(line.split(' ')[-1].rstrip('\n'))-1
|
|
59
|
+
|
|
60
|
+
half_helices.append((i,j))
|
|
61
|
+
half_helices.append((k,l))
|
|
62
|
+
helices.append((name,(i,j),(k,l)))
|
|
63
|
+
|
|
64
|
+
partner[(k,l)] = (i,j)
|
|
65
|
+
|
|
66
|
+
half_helices = sorted(half_helices, key=lambda x : x[0])
|
|
67
|
+
|
|
68
|
+
cnt1 = 1
|
|
69
|
+
cnt2 = 2
|
|
70
|
+
|
|
71
|
+
import copy
|
|
72
|
+
|
|
73
|
+
pos = {}
|
|
74
|
+
|
|
75
|
+
for k in range(len(half_helices)):
|
|
76
|
+
|
|
77
|
+
i,j = half_helices[k]
|
|
78
|
+
pos[(i,j)] = (copy.copy(cnt1), copy.copy(cnt2))
|
|
79
|
+
|
|
80
|
+
cnt1 += 1
|
|
81
|
+
cnt2 += 1
|
|
82
|
+
|
|
83
|
+
# end of figuring out mapping
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
cons_column_writes = [] # list of dicts: column -> symbol. should be exactly one symbol for every column
|
|
87
|
+
insert_writes = [] # list of dicts: column -> the insert written between column and column+1
|
|
88
|
+
num_inserts = {} # column: length of longest insert between column and column+1
|
|
89
|
+
ids = []
|
|
90
|
+
|
|
91
|
+
num_cons_cols = 0
|
|
92
|
+
for k, c in enumerate(list(rf_line)):
|
|
93
|
+
if c != '.':
|
|
94
|
+
num_cons_cols += 1
|
|
95
|
+
|
|
96
|
+
## retrieve HMM model
|
|
97
|
+
#print("READING HMM")
|
|
98
|
+
#start_reading = False
|
|
99
|
+
#for line in open(args.pkcm_file).readlines():
|
|
100
|
+
# if line.startswith("## FULL HMM"):
|
|
101
|
+
# start_reading = True
|
|
102
|
+
# continue
|
|
103
|
+
# if start_reading:
|
|
104
|
+
# if line.startswith("transition probabilities"):
|
|
105
|
+
# transition_probabilities_hmm = parse_transition_probas_hmm(line)
|
|
106
|
+
# if line.startswith("match_emissions probabilities"):
|
|
107
|
+
# match_emissions_probabilities_hmm = parse_transition_probas_hmm(line)
|
|
108
|
+
# if line.startswith("insert_emissions probabilities"):
|
|
109
|
+
# insert_emissions_probabilities_hmm = parse_transition_probas_hmm(line)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# map id -> seq
|
|
113
|
+
id2seq = {}
|
|
114
|
+
|
|
115
|
+
# loop over input sequences
|
|
116
|
+
cnt = 0
|
|
117
|
+
times = []
|
|
118
|
+
is_too_long_for_beta = []
|
|
119
|
+
for record in SeqIO.parse(args.fasta_file, "fasta"):
|
|
120
|
+
if cnt >= args.limit_num_sequences:
|
|
121
|
+
break
|
|
122
|
+
cnt += 1
|
|
123
|
+
|
|
124
|
+
ids.append(record.id)
|
|
125
|
+
id2seq[record.id] = str(record.seq)
|
|
126
|
+
|
|
127
|
+
print("aligning",record.id)
|
|
128
|
+
outfile.write(" ".join(["aligning", record.id,'\n']))
|
|
129
|
+
t0 = time.time()
|
|
130
|
+
try:
|
|
131
|
+
opt, mapping, state_lists = rust_align_wrapper(args.pkcm_file,
|
|
132
|
+
args.td_file,
|
|
133
|
+
str(record.seq),
|
|
134
|
+
args.beta,
|
|
135
|
+
args.tau,
|
|
136
|
+
args.exact,
|
|
137
|
+
args.dense,
|
|
138
|
+
args.noqdb,
|
|
139
|
+
args.nohmm)
|
|
140
|
+
except QdbError as e:
|
|
141
|
+
print(str(e))
|
|
142
|
+
is_too_long_for_beta.append(record.id)
|
|
143
|
+
continue
|
|
144
|
+
|
|
145
|
+
except HmmQdbError as e:
|
|
146
|
+
print(str(e))
|
|
147
|
+
is_too_long_for_beta.append(record.id)
|
|
148
|
+
continue
|
|
149
|
+
|
|
150
|
+
times.append(time.time()-t0)
|
|
151
|
+
|
|
152
|
+
# corrected score using NULL random model
|
|
153
|
+
score = opt+2*len(str(record.seq))
|
|
154
|
+
|
|
155
|
+
print("computation time ",time.time()-t0)
|
|
156
|
+
print("id, opt, score", record.id, opt, score)
|
|
157
|
+
print("mapping ", mapping)
|
|
158
|
+
outfile.write(f"id {record.id} score {score} computation_time {time.time()-t0}\n")
|
|
159
|
+
|
|
160
|
+
# null3 proba
|
|
161
|
+
nA = float(record.seq.count('A'))
|
|
162
|
+
nU = float(record.seq.count('U'))
|
|
163
|
+
nG = float(record.seq.count('G'))
|
|
164
|
+
nC = float(record.seq.count('C'))
|
|
165
|
+
nTotal = nA+nU+nG+nC
|
|
166
|
+
|
|
167
|
+
null3_log_prob = 0.0
|
|
168
|
+
for c in ['A','U','G','C']:
|
|
169
|
+
n = float(record.seq.count(c))
|
|
170
|
+
if n > 0:
|
|
171
|
+
null3_log_prob += n*np.log2((n/nTotal))
|
|
172
|
+
|
|
173
|
+
seq_list = list(str(record.seq))
|
|
174
|
+
|
|
175
|
+
# converting to dict
|
|
176
|
+
map_dict = {}
|
|
177
|
+
for key,val in mapping:
|
|
178
|
+
map_dict[key] = val
|
|
179
|
+
|
|
180
|
+
cons_column_write = {}
|
|
181
|
+
insert_write = {}
|
|
182
|
+
for col_index in range(num_cons_cols+2):
|
|
183
|
+
insert_write[col_index] = [""]
|
|
184
|
+
|
|
185
|
+
for helix_index in range(len(helices)):
|
|
186
|
+
|
|
187
|
+
i,j = helices[helix_index][1]
|
|
188
|
+
k,l = helices[helix_index][2]
|
|
189
|
+
|
|
190
|
+
pos_i,pos_j = pos[(i,j)]
|
|
191
|
+
pos_k,pos_l = pos[(k,l)]
|
|
192
|
+
|
|
193
|
+
start_L,end_L = map_dict[pos_i]+1, map_dict[pos_j]
|
|
194
|
+
start_R,end_R = map_dict[pos_k]+1, map_dict[pos_l]
|
|
195
|
+
|
|
196
|
+
var_L = start_L-1
|
|
197
|
+
var_R = end_R-1
|
|
198
|
+
|
|
199
|
+
# shifting
|
|
200
|
+
i += 1
|
|
201
|
+
l += 1
|
|
202
|
+
|
|
203
|
+
#print("starting indices",i,l)
|
|
204
|
+
#print("map_dict", map_dict)
|
|
205
|
+
#print(state_lists[helix_index])
|
|
206
|
+
#print("left seq", "".join(seq_list[start_L-1:end_L]),"right seq","".join(seq_list[start_R-1:end_R]))
|
|
207
|
+
|
|
208
|
+
def left_insert_pos(state, num_cons_cols):
|
|
209
|
+
if state.startswith("ROOT"):
|
|
210
|
+
return 0
|
|
211
|
+
try:
|
|
212
|
+
i = int(state.split('_')[1])
|
|
213
|
+
if state.startswith("BEGR"):
|
|
214
|
+
return i-1
|
|
215
|
+
except:
|
|
216
|
+
assert(state.split('_')[1]=='-')
|
|
217
|
+
i = 0
|
|
218
|
+
return i
|
|
219
|
+
|
|
220
|
+
def right_insert_pos(state, num_cons_cols):
|
|
221
|
+
if state.startswith("ROOT"):
|
|
222
|
+
return num_cons_cols
|
|
223
|
+
try:
|
|
224
|
+
i = int(state.split('_')[2])-1
|
|
225
|
+
except:
|
|
226
|
+
assert(state.split('_')[2]=='-')
|
|
227
|
+
i = num_cons_cols
|
|
228
|
+
return i
|
|
229
|
+
|
|
230
|
+
for state in state_lists[helix_index]:
|
|
231
|
+
# in what follows: i and l track the cons positions
|
|
232
|
+
# var_L and var_R track the sequence positions
|
|
233
|
+
if state.endswith("IL"):
|
|
234
|
+
insert_write[left_insert_pos(state, num_cons_cols)].append(seq_list[var_L])
|
|
235
|
+
var_L += 1
|
|
236
|
+
continue
|
|
237
|
+
|
|
238
|
+
if state.endswith("IR"):
|
|
239
|
+
insert_write[right_insert_pos(state, num_cons_cols)] = [seq_list[var_R]] + insert_write[right_insert_pos(state, num_cons_cols)]
|
|
240
|
+
var_R -= 1
|
|
241
|
+
continue
|
|
242
|
+
|
|
243
|
+
if state.startswith("MATP"):
|
|
244
|
+
if state.endswith("MP"):
|
|
245
|
+
cons_column_write[i] = seq_list[var_L]
|
|
246
|
+
cons_column_write[l] = seq_list[var_R]
|
|
247
|
+
|
|
248
|
+
var_L += 1
|
|
249
|
+
var_R -= 1
|
|
250
|
+
|
|
251
|
+
if state.endswith("ML"):
|
|
252
|
+
cons_column_write[i] = seq_list[var_L]
|
|
253
|
+
cons_column_write[l] = "-"
|
|
254
|
+
|
|
255
|
+
# no var_R decrement
|
|
256
|
+
var_L += 1
|
|
257
|
+
|
|
258
|
+
if state.endswith("MR"):
|
|
259
|
+
cons_column_write[i] = "-"
|
|
260
|
+
cons_column_write[l] = seq_list[var_R]
|
|
261
|
+
|
|
262
|
+
var_R -= 1
|
|
263
|
+
|
|
264
|
+
if state.endswith("D"):
|
|
265
|
+
cons_column_write[i] = "-"
|
|
266
|
+
cons_column_write[l] = "-"
|
|
267
|
+
|
|
268
|
+
# for matp node: in any case we write both left/right
|
|
269
|
+
# consensus columns
|
|
270
|
+
i += 1
|
|
271
|
+
l -= 1
|
|
272
|
+
|
|
273
|
+
if state.startswith("MATL"):
|
|
274
|
+
if state.endswith("ML"):
|
|
275
|
+
cons_column_write[i] = seq_list[var_L]
|
|
276
|
+
var_L += 1
|
|
277
|
+
|
|
278
|
+
if state.endswith("D"):
|
|
279
|
+
cons_column_write[i] = "-"
|
|
280
|
+
|
|
281
|
+
i += 1
|
|
282
|
+
|
|
283
|
+
if state.startswith("MATR"):
|
|
284
|
+
if state.endswith("MR"):
|
|
285
|
+
cons_column_write[l] = seq_list[var_R]
|
|
286
|
+
var_R -= 1
|
|
287
|
+
|
|
288
|
+
if state.endswith("D"):
|
|
289
|
+
cons_column_write[l] = "-"
|
|
290
|
+
|
|
291
|
+
l -= 1
|
|
292
|
+
|
|
293
|
+
cons_column_writes.append(cons_column_write)
|
|
294
|
+
insert_writes.append(insert_write)
|
|
295
|
+
|
|
296
|
+
for k in range(num_cons_cols+2):
|
|
297
|
+
max_ins = 0
|
|
298
|
+
for insert_write in insert_writes:
|
|
299
|
+
max_ins = max(max_ins,len(insert_write[k])-1) # -1 because there is ""
|
|
300
|
+
|
|
301
|
+
num_inserts[k] = max_ins
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
max_id_len = 0
|
|
305
|
+
for identifier in ids+['ss_cons','rf_line']:
|
|
306
|
+
max_id_len = max(len(identifier)+1, max_id_len)
|
|
307
|
+
|
|
308
|
+
def pad_with_spaces(s, L):
|
|
309
|
+
while len(s) < L:
|
|
310
|
+
s += ' '
|
|
311
|
+
return s
|
|
312
|
+
|
|
313
|
+
# TODO: right-inserts should be padded on the left, not
|
|
314
|
+
# the right
|
|
315
|
+
def pad_with_gaps(s, L):
|
|
316
|
+
while len(s) < L:
|
|
317
|
+
s += '-'
|
|
318
|
+
return s
|
|
319
|
+
|
|
320
|
+
def pad_with_dots(s, L):
|
|
321
|
+
while len(s) < L:
|
|
322
|
+
s += '.'
|
|
323
|
+
return s
|
|
324
|
+
|
|
325
|
+
def print_aligned_seq(identifier, num_cons_cols, insert_write, cons_column_write, num_inserts, max_id_len):
|
|
326
|
+
|
|
327
|
+
# print("cons write", sorted(cons_column_write.items(),key = lambda x : x[0]))
|
|
328
|
+
# print("insert write", insert_write)
|
|
329
|
+
|
|
330
|
+
# also storing the seq for validation purposes
|
|
331
|
+
seq = ""
|
|
332
|
+
|
|
333
|
+
print(pad_with_spaces(identifier, max_id_len),end='')
|
|
334
|
+
outfile.write(pad_with_spaces(identifier, max_id_len))
|
|
335
|
+
for col_index in range(num_cons_cols+2):
|
|
336
|
+
if col_index > 0 and col_index <= num_cons_cols:
|
|
337
|
+
print(cons_column_write[col_index], end='')
|
|
338
|
+
outfile.write(cons_column_write[col_index])
|
|
339
|
+
seq += cons_column_write[col_index]
|
|
340
|
+
|
|
341
|
+
print(pad_with_gaps(''.join(insert_write[col_index]), num_inserts[col_index]),end='')
|
|
342
|
+
outfile.write(pad_with_gaps(''.join(insert_write[col_index]), num_inserts[col_index]))
|
|
343
|
+
seq += ''.join(insert_write[col_index])
|
|
344
|
+
|
|
345
|
+
print("")
|
|
346
|
+
outfile.write('\n')
|
|
347
|
+
return seq
|
|
348
|
+
|
|
349
|
+
def print_aligned_cons(identifier, num_cons_cols, insert_write, cons_column_write, num_inserts, max_id_len):
|
|
350
|
+
print(pad_with_spaces(identifier, max_id_len),end='')
|
|
351
|
+
outfile.write(pad_with_spaces(identifier, max_id_len))
|
|
352
|
+
for col_index in range(num_cons_cols+2):
|
|
353
|
+
if col_index > 0 and col_index <= num_cons_cols:
|
|
354
|
+
print(cons_column_write[col_index], end='')
|
|
355
|
+
outfile.write(cons_column_write[col_index])
|
|
356
|
+
|
|
357
|
+
print(pad_with_dots(''.join(insert_write[col_index]), num_inserts[col_index]),end='')
|
|
358
|
+
outfile.write(pad_with_dots(''.join(insert_write[col_index]), num_inserts[col_index]))
|
|
359
|
+
|
|
360
|
+
print("")
|
|
361
|
+
outfile.write('\n')
|
|
362
|
+
|
|
363
|
+
# making objects with good format for ss_cons and gc_rf
|
|
364
|
+
ss_cons_col_write = {}
|
|
365
|
+
ss_cons_insert_write = {}
|
|
366
|
+
rf_line_col_write = {}
|
|
367
|
+
rf_line_insert_write = {}
|
|
368
|
+
curr_insert = ""
|
|
369
|
+
tracker = 0
|
|
370
|
+
|
|
371
|
+
for col_index in range(num_cons_cols+2):
|
|
372
|
+
ss_cons_insert_write[col_index] = ""
|
|
373
|
+
rf_line_insert_write[col_index] = ""
|
|
374
|
+
|
|
375
|
+
col_index = 0
|
|
376
|
+
for k,c in enumerate(list(rf_line)):
|
|
377
|
+
if c!='.':
|
|
378
|
+
ss_cons_col_write[col_index+1] = ss_cons[k]
|
|
379
|
+
rf_line_col_write[col_index+1] = rf_line[k]
|
|
380
|
+
col_index += 1
|
|
381
|
+
|
|
382
|
+
print("average single-sequence alignment time ", np.mean(times),"seconds")
|
|
383
|
+
outfile.write(" ".join(["average single-sequence alignment time ", str(np.mean(times)),"seconds\n"]))
|
|
384
|
+
|
|
385
|
+
# final print
|
|
386
|
+
print_aligned_cons("ss_cons", num_cons_cols, ss_cons_insert_write, ss_cons_col_write, num_inserts, max_id_len)
|
|
387
|
+
print_aligned_cons("rf_line", num_cons_cols, rf_line_insert_write, rf_line_col_write, num_inserts, max_id_len)
|
|
388
|
+
|
|
389
|
+
ids = [identifier for identifier in ids if identifier not in is_too_long_for_beta]
|
|
390
|
+
|
|
391
|
+
for num,identifier in enumerate(ids):
|
|
392
|
+
reconstructed_seq = print_aligned_seq(identifier, num_cons_cols, insert_writes[num], cons_column_writes[num], num_inserts, max_id_len)
|
|
393
|
+
reconstructed_seq = reconstructed_seq.replace('-','')
|
|
394
|
+
#print("for id", identifier, "reconstructed seq is ", reconstructed_seq)
|
|
395
|
+
#print(" whereas seq is ", id2seq[identifier])
|
|
396
|
+
assert(reconstructed_seq==id2seq[identifier])
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
from pkcm.utils import Node, Helix, adhoc_helix_decomposition, get_sscons, preambule_stk_file, alignment_lines_stk_file, filter_alignment, homemade_cmbuild, read_pkcm_lines_infernal_format, equalise_whitespace
|
|
3
|
+
import subprocess
|
|
4
|
+
import tempfile
|
|
5
|
+
import sys
|
|
6
|
+
import argparse
|
|
7
|
+
|
|
8
|
+
# parser configuration
|
|
9
|
+
parser = argparse.ArgumentParser(description='Builds a pseudoknotted covariance model (.pkcm file) from a multiple sequence alignment. Also outputs a .helix file specifying how the consensus structure was decomposed into helices.')
|
|
10
|
+
|
|
11
|
+
# main arguments
|
|
12
|
+
parser.add_argument('output_pkcm_filename', type=str, help='name of the file in which to write the pseudoknotted covariance model.')
|
|
13
|
+
parser.add_argument('inputfile_msa_name', type=str, help='name of the input multiple sequence alignment')
|
|
14
|
+
|
|
15
|
+
# optional arguments
|
|
16
|
+
parser.add_argument('--dump-helix-alignments',
|
|
17
|
+
type=str,
|
|
18
|
+
help="optionally, directory in which to dump alignment files of each helix.",
|
|
19
|
+
dest='dump_helix_alignments',
|
|
20
|
+
metavar='DIR',
|
|
21
|
+
required=False)
|
|
22
|
+
|
|
23
|
+
parser.add_argument('--ignore-pk',
|
|
24
|
+
help="Ignore the pk helices (those with Aa,Bb,Cc...)",
|
|
25
|
+
dest='ignore_pk',
|
|
26
|
+
action='store_true',
|
|
27
|
+
required=False)
|
|
28
|
+
|
|
29
|
+
parser.add_argument('--use-infernal-format',
|
|
30
|
+
help="If present, use infernal (cmbuild) for each helix. Instead of simple Laplace smoothing and ``home-made'' format.",
|
|
31
|
+
dest='use_infernal',
|
|
32
|
+
action='store_true',
|
|
33
|
+
required=False)
|
|
34
|
+
|
|
35
|
+
# parsing args
|
|
36
|
+
args = parser.parse_args()
|
|
37
|
+
|
|
38
|
+
msa_filename = args.inputfile_msa_name
|
|
39
|
+
cm_outname = args.output_pkcm_filename
|
|
40
|
+
try:
|
|
41
|
+
helix_msa_dir = args.dump_helix_alignments
|
|
42
|
+
except AttributeError:
|
|
43
|
+
helix_msa_dir = None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# getting ss_cons
|
|
47
|
+
ss_cons, rf_line = get_sscons(open(msa_filename).readlines(), ignore_pk=args.ignore_pk)
|
|
48
|
+
rf_line = rf_line.replace('.','')
|
|
49
|
+
N = len(ss_cons)
|
|
50
|
+
|
|
51
|
+
# helix decomposition
|
|
52
|
+
helix_list = adhoc_helix_decomposition(ss_cons)
|
|
53
|
+
|
|
54
|
+
pkcm_file = open(cm_outname,'w')
|
|
55
|
+
|
|
56
|
+
if args.use_infernal:
|
|
57
|
+
print('#USING_CMBUILD_BACKEND=true',file=pkcm_file)
|
|
58
|
+
ss_cons_spacer = '<<<.>>>'
|
|
59
|
+
gc_rf_spacer = '..GAC..'
|
|
60
|
+
else:
|
|
61
|
+
print('#USING_CMBUILD_BACKEND=false',file=pkcm_file)
|
|
62
|
+
ss_cons_spacer = '||'
|
|
63
|
+
gc_rf_spacer = '||'
|
|
64
|
+
|
|
65
|
+
# storing helix decomposition description
|
|
66
|
+
print("## BEGIN HELIX ARRANGEMENT DESCRIPTION ##", file=pkcm_file)
|
|
67
|
+
for helix in helix_list:
|
|
68
|
+
print(helix.name, helix.parenthesis_system, helix.print_boundaries(), file=pkcm_file)
|
|
69
|
+
for node in helix.nodes:
|
|
70
|
+
print(node.type, node.left_end, node.right_end,file=pkcm_file)
|
|
71
|
+
print('-'*10,file=pkcm_file)
|
|
72
|
+
print("## END HELIX ARRANGEMENT DESCRIPTION ##", file=pkcm_file)
|
|
73
|
+
|
|
74
|
+
preambule = preambule_stk_file(open(msa_filename).readlines())
|
|
75
|
+
alignment_lines = alignment_lines_stk_file(open(msa_filename).readlines(),ignore_pk=args.ignore_pk)
|
|
76
|
+
|
|
77
|
+
all_lines = []
|
|
78
|
+
|
|
79
|
+
cm_lines_per_helix = {}
|
|
80
|
+
|
|
81
|
+
if not args.use_infernal:
|
|
82
|
+
print("## BEGIN PKCM ##", file=pkcm_file)
|
|
83
|
+
|
|
84
|
+
for helix in helix_list:
|
|
85
|
+
|
|
86
|
+
# isolating the part of the alignment concerning the helix
|
|
87
|
+
alignment_file = tempfile.NamedTemporaryFile()
|
|
88
|
+
filter_alignment(preambule, alignment_lines, helix, alignment_file,ss_cons_spacer=ss_cons_spacer,gc_rf_spacer=gc_rf_spacer)
|
|
89
|
+
alignment_file.seek(0)
|
|
90
|
+
if helix_msa_dir is not None:
|
|
91
|
+
helix_msa_file = open(helix_msa_dir+msa_filename.split('.')[0].split('/')[-1]+'_'+helix.name+'.seed','w')
|
|
92
|
+
for line in alignment_file.readlines():
|
|
93
|
+
helix_msa_file.write(line.decode('utf-8'))
|
|
94
|
+
alignment_file.seek(0)
|
|
95
|
+
|
|
96
|
+
if args.use_infernal:
|
|
97
|
+
# file to store the resulting covariance model for that helix
|
|
98
|
+
helix_cm_file = tempfile.NamedTemporaryFile()
|
|
99
|
+
|
|
100
|
+
print("input given to cmbuild:")
|
|
101
|
+
for line in alignment_file.readlines():
|
|
102
|
+
print(line)
|
|
103
|
+
alignment_file.seek(0)
|
|
104
|
+
|
|
105
|
+
# infernal run
|
|
106
|
+
cmd_args = ['cmbuild','--iins','--iflank','--hand','-F',helix_cm_file.name,alignment_file.name]
|
|
107
|
+
subprocess.run(cmd_args, check=True)
|
|
108
|
+
|
|
109
|
+
# dumping into overall pkcm file
|
|
110
|
+
helix_cm_file.seek(0)
|
|
111
|
+
cm_lines_per_helix[helix.name] = list(helix_cm_file.readlines())
|
|
112
|
+
|
|
113
|
+
all_lines.append('## CM FILE for '+helix.name)
|
|
114
|
+
for line in cm_lines_per_helix[helix.name]:
|
|
115
|
+
print(line)
|
|
116
|
+
all_lines.append(line.decode("utf-8"))
|
|
117
|
+
|
|
118
|
+
else:
|
|
119
|
+
print('-----------',file=pkcm_file)
|
|
120
|
+
print(helix.name, helix.parenthesis_system, helix.print_boundaries(), file=pkcm_file)
|
|
121
|
+
|
|
122
|
+
transition_edge_list, emission_list = homemade_cmbuild(alignment_file, helix)
|
|
123
|
+
|
|
124
|
+
for state1, state2, weight in transition_edge_list:
|
|
125
|
+
print('transition',state1,'->',state2,weight,file=pkcm_file)
|
|
126
|
+
|
|
127
|
+
for state, nucl, weight in emission_list:
|
|
128
|
+
print('emission',state,'->', nucl, weight, file=pkcm_file)
|
|
129
|
+
|
|
130
|
+
if not args.use_infernal:
|
|
131
|
+
print("## END PKCM ##", file=pkcm_file)
|
|
132
|
+
|
|
133
|
+
if args.use_infernal:
|
|
134
|
+
transition_probabilities, emission_probabilities = read_pkcm_lines_infernal_format(all_lines, helix_list)
|
|
135
|
+
print("## BEGIN PKCM ##", file=pkcm_file)
|
|
136
|
+
for helix in helix_list:
|
|
137
|
+
print('-----------',file=pkcm_file)
|
|
138
|
+
print(helix.name, helix.parenthesis_system, helix.print_boundaries(), file=pkcm_file)
|
|
139
|
+
for key, value in transition_probabilities[helix.name].items():
|
|
140
|
+
print('transition',key,value,file=pkcm_file)
|
|
141
|
+
for state, value in emission_probabilities[helix.name].items():
|
|
142
|
+
for nucl, weight in value.items():
|
|
143
|
+
print('emission',state,'->',nucl,weight,file=pkcm_file)
|
|
144
|
+
print("## END PKCM ##", file=pkcm_file)
|
|
145
|
+
print("## BEGIN INDIVIDUAL CM FILES", file=pkcm_file)
|
|
146
|
+
for helix in helix_list:
|
|
147
|
+
for line in cm_lines_per_helix[helix.name]:
|
|
148
|
+
print(line.decode("utf-8"),end='',file=pkcm_file)
|
|
149
|
+
print("## END INDIVIDUAL CM FILES", file=pkcm_file)
|
|
150
|
+
else:
|
|
151
|
+
print("## END PKCM ##", file=pkcm_file)
|
|
152
|
+
|
|
153
|
+
rf_line, ss_cons = equalise_whitespace(rf_line, '#=GC SS_cons '+ss_cons)
|
|
154
|
+
|
|
155
|
+
import pyhmmer
|
|
156
|
+
from pyhmmer import easel, plan7
|
|
157
|
+
|
|
158
|
+
alphabet = easel.Alphabet.rna()
|
|
159
|
+
|
|
160
|
+
with easel.MSAFile(args.inputfile_msa_name, digital=False) as msa_file:
|
|
161
|
+
msa = next(msa_file)
|
|
162
|
+
|
|
163
|
+
digital_msa = msa.digitize(alphabet)
|
|
164
|
+
|
|
165
|
+
builder = plan7.Builder(alphabet)
|
|
166
|
+
background = plan7.Background(alphabet)
|
|
167
|
+
|
|
168
|
+
hmm, profile, optimized_profile = builder.build_msa(
|
|
169
|
+
digital_msa, background
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
print("## FULL HMM ", file=pkcm_file)
|
|
173
|
+
print("transition probabilities", hmm.transition_probabilities, file=pkcm_file)
|
|
174
|
+
print("match_emission probabilities", hmm.match_emissions, file=pkcm_file)
|
|
175
|
+
print("insert_emission probabilities", hmm.insert_emissions, file=pkcm_file)
|
|
176
|
+
print("## END FULL HMM", file=pkcm_file)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
pkcm_file.write('### CONSENSUS INFO'+'\n')
|
|
180
|
+
pkcm_file.write(ss_cons+'\n')
|
|
181
|
+
pkcm_file.write(rf_line+'\n')
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
import random
|
|
3
|
+
from pkcm.utils import read_helices_in_file, read_pkcm_file
|
|
4
|
+
import tempfile
|
|
5
|
+
import argparse
|
|
6
|
+
import subprocess
|
|
7
|
+
|
|
8
|
+
parser = argparse.ArgumentParser(description="Emits sequences from a pseudoknotted covariance model.")
|
|
9
|
+
|
|
10
|
+
parser.add_argument('pkcm_file', type=str, help='.pkcm file to emit from')
|
|
11
|
+
parser.add_argument('-N', type=int, help="number of samples to get. (default=10)", default=10)
|
|
12
|
+
|
|
13
|
+
args = parser.parse_args()
|
|
14
|
+
N = args.N
|
|
15
|
+
|
|
16
|
+
helices = read_helices_in_file(args.pkcm_file)
|
|
17
|
+
|
|
18
|
+
transition_probabilities, in_ngbh, emission_probabilities, start_state = read_pkcm_file(open(args.pkcm_file).readlines())
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def sample(list_of_outcomes):
|
|
22
|
+
weight_list = [2**w for _,w in list_of_outcomes]
|
|
23
|
+
cumulative_weight = 0
|
|
24
|
+
cumulative_weights = []
|
|
25
|
+
for k, p in enumerate(weight_list):
|
|
26
|
+
cumulative_weight += p
|
|
27
|
+
cumulative_weights.append(cumulative_weight)
|
|
28
|
+
|
|
29
|
+
r = random.random()
|
|
30
|
+
|
|
31
|
+
for k, cw in enumerate(cumulative_weights):
|
|
32
|
+
if r-cw < 0:
|
|
33
|
+
return list_of_outcomes[k][0]
|
|
34
|
+
|
|
35
|
+
return list_of_outcomes[-1][0] # sometimes we can have
|
|
36
|
+
# cumulative weights
|
|
37
|
+
# not exactly summing to 1
|
|
38
|
+
# because of precision errors
|
|
39
|
+
|
|
40
|
+
for _ in range(N):
|
|
41
|
+
sequences = []
|
|
42
|
+
for helix in helices:
|
|
43
|
+
try:
|
|
44
|
+
state = start_state[helix.name]
|
|
45
|
+
except KeyError:
|
|
46
|
+
no_in = [s for s in transition_probabilities[helix.name].keys() if s not in in_ngbh[helix.name].keys()]
|
|
47
|
+
state = random.choice(no_in)
|
|
48
|
+
left_seq = ""
|
|
49
|
+
right_seq = ""
|
|
50
|
+
while True:
|
|
51
|
+
if state in emission_probabilities[helix.name].keys():
|
|
52
|
+
outcome = sample(emission_probabilities[helix.name][state])
|
|
53
|
+
if state.endswith('L'):
|
|
54
|
+
left_seq += outcome
|
|
55
|
+
if state.endswith('R'):
|
|
56
|
+
right_seq = outcome + right_seq
|
|
57
|
+
if state.endswith('P'):
|
|
58
|
+
left_seq += outcome[0]
|
|
59
|
+
right_seq = outcome[0] + right_seq
|
|
60
|
+
if state in transition_probabilities[helix.name].keys():
|
|
61
|
+
state = sample(transition_probabilities[helix.name][state])
|
|
62
|
+
else:
|
|
63
|
+
break
|
|
64
|
+
sequences.append((helix.outer_left, left_seq))
|
|
65
|
+
sequences.append((helix.inner_left, right_seq))
|
|
66
|
+
|
|
67
|
+
print(''.join([s[1] for s in sorted(sequences, key=lambda x : x[0])]))
|
|
68
|
+
|