eFISHent 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (267) hide show
  1. eFISHent/__init__.py +23 -0
  2. eFISHent/__main__.py +6 -0
  3. eFISHent/alignment.py +336 -0
  4. eFISHent/analyze.py +246 -0
  5. eFISHent/basic_filtering.py +106 -0
  6. eFISHent/cleanup.py +163 -0
  7. eFISHent/cli.py +214 -0
  8. eFISHent/config.py +181 -0
  9. eFISHent/constants.py +95 -0
  10. eFISHent/data_tables/b-test.coaxial.dg +111 -0
  11. eFISHent/data_tables/b-test.coaxial.dh +111 -0
  12. eFISHent/data_tables/b-test.coaxstack.dg +349 -0
  13. eFISHent/data_tables/b-test.coaxstack.dh +349 -0
  14. eFISHent/data_tables/b-test.dangle.dg +499 -0
  15. eFISHent/data_tables/b-test.dangle.dh +499 -0
  16. eFISHent/data_tables/b-test.dynalignmiscloop.dg +62 -0
  17. eFISHent/data_tables/b-test.hexaloop.dg +26 -0
  18. eFISHent/data_tables/b-test.hexaloop.dh +26 -0
  19. eFISHent/data_tables/b-test.int11.dg +1047 -0
  20. eFISHent/data_tables/b-test.int11.dh +895 -0
  21. eFISHent/data_tables/b-test.int21.dg +5897 -0
  22. eFISHent/data_tables/b-test.int21.dh +3583 -0
  23. eFISHent/data_tables/b-test.int22.dg +2239 -0
  24. eFISHent/data_tables/b-test.int22.dh +2239 -0
  25. eFISHent/data_tables/b-test.loop.dg +34 -0
  26. eFISHent/data_tables/b-test.miscloop.dg +66 -0
  27. eFISHent/data_tables/b-test.miscloop.dh +66 -0
  28. eFISHent/data_tables/b-test.specification.dat +64 -0
  29. eFISHent/data_tables/b-test.stack.dg +111 -0
  30. eFISHent/data_tables/b-test.stack.dh +111 -0
  31. eFISHent/data_tables/b-test.tloop.dg +34 -0
  32. eFISHent/data_tables/b-test.tloop.dh +34 -0
  33. eFISHent/data_tables/b-test.triloop.dg +8 -0
  34. eFISHent/data_tables/b-test.triloop.dh +8 -0
  35. eFISHent/data_tables/b-test.tstack.dg +111 -0
  36. eFISHent/data_tables/b-test.tstack.dh +111 -0
  37. eFISHent/data_tables/b-test.tstackcoax.dg +111 -0
  38. eFISHent/data_tables/b-test.tstackcoax.dh +111 -0
  39. eFISHent/data_tables/b-test.tstackh.dg +111 -0
  40. eFISHent/data_tables/b-test.tstackh.dh +111 -0
  41. eFISHent/data_tables/b-test.tstacki.dg +111 -0
  42. eFISHent/data_tables/b-test.tstacki.dh +111 -0
  43. eFISHent/data_tables/b-test.tstacki1n.dg +111 -0
  44. eFISHent/data_tables/b-test.tstacki1n.dh +111 -0
  45. eFISHent/data_tables/b-test.tstacki23.dg +111 -0
  46. eFISHent/data_tables/b-test.tstacki23.dh +111 -0
  47. eFISHent/data_tables/b-test.tstackm.dg +111 -0
  48. eFISHent/data_tables/b-test.tstackm.dh +111 -0
  49. eFISHent/data_tables/coaxial.dat +75 -0
  50. eFISHent/data_tables/coaxial.dh +75 -0
  51. eFISHent/data_tables/coaxstack.dat +75 -0
  52. eFISHent/data_tables/coaxstack.dh +75 -0
  53. eFISHent/data_tables/dangle.dat +89 -0
  54. eFISHent/data_tables/dangle.dh +89 -0
  55. eFISHent/data_tables/design.DNA.Helices.dat +2000 -0
  56. eFISHent/data_tables/design.DNA.Loops.dat +2000 -0
  57. eFISHent/data_tables/design.RNA.Helices.dat +2000 -0
  58. eFISHent/data_tables/design.RNA.Loops.dat +2075 -0
  59. eFISHent/data_tables/dna.coaxial.dg +102 -0
  60. eFISHent/data_tables/dna.coaxial.dh +102 -0
  61. eFISHent/data_tables/dna.coaxstack.dg +244 -0
  62. eFISHent/data_tables/dna.coaxstack.dh +244 -0
  63. eFISHent/data_tables/dna.dangle.dg +363 -0
  64. eFISHent/data_tables/dna.dangle.dh +363 -0
  65. eFISHent/data_tables/dna.dynalignmiscloop.dg +54 -0
  66. eFISHent/data_tables/dna.dynalignmiscloop.dh +67 -0
  67. eFISHent/data_tables/dna.hexaloop.dg +2 -0
  68. eFISHent/data_tables/dna.hexaloop.dh +2 -0
  69. eFISHent/data_tables/dna.int11.dg +601 -0
  70. eFISHent/data_tables/dna.int11.dh +601 -0
  71. eFISHent/data_tables/dna.int21.dg +2329 -0
  72. eFISHent/data_tables/dna.int21.dh +2329 -0
  73. eFISHent/data_tables/dna.int22.dg +967 -0
  74. eFISHent/data_tables/dna.int22.dh +967 -0
  75. eFISHent/data_tables/dna.loop.dg +34 -0
  76. eFISHent/data_tables/dna.loop.dh +34 -0
  77. eFISHent/data_tables/dna.miscloop.dg +67 -0
  78. eFISHent/data_tables/dna.miscloop.dh +67 -0
  79. eFISHent/data_tables/dna.specification.dat +58 -0
  80. eFISHent/data_tables/dna.stack.dg +102 -0
  81. eFISHent/data_tables/dna.stack.dh +102 -0
  82. eFISHent/data_tables/dna.tloop.dg +9 -0
  83. eFISHent/data_tables/dna.tloop.dh +8 -0
  84. eFISHent/data_tables/dna.triloop.dg +5 -0
  85. eFISHent/data_tables/dna.triloop.dh +5 -0
  86. eFISHent/data_tables/dna.tstack.dg +102 -0
  87. eFISHent/data_tables/dna.tstack.dh +102 -0
  88. eFISHent/data_tables/dna.tstackcoax.dg +102 -0
  89. eFISHent/data_tables/dna.tstackcoax.dh +102 -0
  90. eFISHent/data_tables/dna.tstackh.dg +102 -0
  91. eFISHent/data_tables/dna.tstackh.dh +102 -0
  92. eFISHent/data_tables/dna.tstacki.dg +102 -0
  93. eFISHent/data_tables/dna.tstacki.dh +102 -0
  94. eFISHent/data_tables/dna.tstacki1n.dg +102 -0
  95. eFISHent/data_tables/dna.tstacki1n.dh +102 -0
  96. eFISHent/data_tables/dna.tstacki23.dg +102 -0
  97. eFISHent/data_tables/dna.tstacki23.dh +102 -0
  98. eFISHent/data_tables/dna.tstackm.dg +102 -0
  99. eFISHent/data_tables/dna.tstackm.dh +102 -0
  100. eFISHent/data_tables/dnacoaxial.dat +76 -0
  101. eFISHent/data_tables/dnacoaxial.dh +76 -0
  102. eFISHent/data_tables/dnacoaxstack.dat +75 -0
  103. eFISHent/data_tables/dnacoaxstack.dh +76 -0
  104. eFISHent/data_tables/dnadangle.dat +89 -0
  105. eFISHent/data_tables/dnadangle.dh +89 -0
  106. eFISHent/data_tables/dnadynalignmiscloop.dat +54 -0
  107. eFISHent/data_tables/dnadynalignmiscloop.dh +67 -0
  108. eFISHent/data_tables/dnahexaloop.dat +2 -0
  109. eFISHent/data_tables/dnahexaloop.dh +2 -0
  110. eFISHent/data_tables/dnaint11.dat +112 -0
  111. eFISHent/data_tables/dnaint11.dh +117 -0
  112. eFISHent/data_tables/dnaint21.dat +356 -0
  113. eFISHent/data_tables/dnaint21.dh +355 -0
  114. eFISHent/data_tables/dnaint22.dat +967 -0
  115. eFISHent/data_tables/dnaint22.dh +967 -0
  116. eFISHent/data_tables/dnaloop.dat +34 -0
  117. eFISHent/data_tables/dnaloop.dh +34 -0
  118. eFISHent/data_tables/dnamiscloop.dat +63 -0
  119. eFISHent/data_tables/dnamiscloop.dh +63 -0
  120. eFISHent/data_tables/dnastack.dat +76 -0
  121. eFISHent/data_tables/dnastack.dh +76 -0
  122. eFISHent/data_tables/dnatloop.dat +116 -0
  123. eFISHent/data_tables/dnatloop.dh +14 -0
  124. eFISHent/data_tables/dnatriloop.dat +19 -0
  125. eFISHent/data_tables/dnatriloop.dh +5 -0
  126. eFISHent/data_tables/dnatstack.dat +76 -0
  127. eFISHent/data_tables/dnatstack.dh +76 -0
  128. eFISHent/data_tables/dnatstackcoax.dat +76 -0
  129. eFISHent/data_tables/dnatstackcoax.dh +76 -0
  130. eFISHent/data_tables/dnatstackh.dat +76 -0
  131. eFISHent/data_tables/dnatstackh.dh +76 -0
  132. eFISHent/data_tables/dnatstacki.dat +76 -0
  133. eFISHent/data_tables/dnatstacki.dh +76 -0
  134. eFISHent/data_tables/dnatstacki1n.dat +76 -0
  135. eFISHent/data_tables/dnatstacki1n.dh +76 -0
  136. eFISHent/data_tables/dnatstacki23.dat +76 -0
  137. eFISHent/data_tables/dnatstacki23.dh +76 -0
  138. eFISHent/data_tables/dnatstackm.dat +76 -0
  139. eFISHent/data_tables/dnatstackm.dh +76 -0
  140. eFISHent/data_tables/dynalignmiscloop.dat +62 -0
  141. eFISHent/data_tables/fam_hmm_pars.dat +1 -0
  142. eFISHent/data_tables/helix.dat +5 -0
  143. eFISHent/data_tables/helixdr.dat +5 -0
  144. eFISHent/data_tables/hexaloop.dat +6 -0
  145. eFISHent/data_tables/hexaloop.dh +6 -0
  146. eFISHent/data_tables/int11.dat +113 -0
  147. eFISHent/data_tables/int11.dh +113 -0
  148. eFISHent/data_tables/int21.dat +371 -0
  149. eFISHent/data_tables/int21.dh +371 -0
  150. eFISHent/data_tables/int22-exp.dh +967 -0
  151. eFISHent/data_tables/int22.dat +967 -0
  152. eFISHent/data_tables/int22.dh +967 -0
  153. eFISHent/data_tables/loop.dat +34 -0
  154. eFISHent/data_tables/loop.dh +34 -0
  155. eFISHent/data_tables/m6A.coaxial.dg +134 -0
  156. eFISHent/data_tables/m6A.coaxstack.dg +374 -0
  157. eFISHent/data_tables/m6A.dangle.dg +210 -0
  158. eFISHent/data_tables/m6A.hexaloop.dg +6 -0
  159. eFISHent/data_tables/m6A.int11.dg +1133 -0
  160. eFISHent/data_tables/m6A.int21.dg +5440 -0
  161. eFISHent/data_tables/m6A.int22.dg +2304 -0
  162. eFISHent/data_tables/m6A.loop.dg +34 -0
  163. eFISHent/data_tables/m6A.miscloop.dg +66 -0
  164. eFISHent/data_tables/m6A.specification.dat +64 -0
  165. eFISHent/data_tables/m6A.stack.dg +134 -0
  166. eFISHent/data_tables/m6A.tloop.dg +19 -0
  167. eFISHent/data_tables/m6A.triloop.dg +4 -0
  168. eFISHent/data_tables/m6A.tstack.dg +136 -0
  169. eFISHent/data_tables/m6A.tstackcoax.dg +136 -0
  170. eFISHent/data_tables/m6A.tstackh.dg +137 -0
  171. eFISHent/data_tables/m6A.tstacki.dg +142 -0
  172. eFISHent/data_tables/m6A.tstacki1n.dg +140 -0
  173. eFISHent/data_tables/m6A.tstacki23.dg +144 -0
  174. eFISHent/data_tables/m6A.tstackm.dg +136 -0
  175. eFISHent/data_tables/miscloop.dat +62 -0
  176. eFISHent/data_tables/miscloop.dh +62 -0
  177. eFISHent/data_tables/pseudconst.dat +34 -0
  178. eFISHent/data_tables/rna.coaxial.dg +102 -0
  179. eFISHent/data_tables/rna.coaxial.dh +102 -0
  180. eFISHent/data_tables/rna.coaxstack.dg +244 -0
  181. eFISHent/data_tables/rna.coaxstack.dh +244 -0
  182. eFISHent/data_tables/rna.cov.dg +294 -0
  183. eFISHent/data_tables/rna.cov.dh +294 -0
  184. eFISHent/data_tables/rna.dangle.dg +363 -0
  185. eFISHent/data_tables/rna.dangle.dh +363 -0
  186. eFISHent/data_tables/rna.dynalignmiscloop.dg +62 -0
  187. eFISHent/data_tables/rna.hexaloop.dg +6 -0
  188. eFISHent/data_tables/rna.hexaloop.dh +6 -0
  189. eFISHent/data_tables/rna.int11.dg +601 -0
  190. eFISHent/data_tables/rna.int11.dh +601 -0
  191. eFISHent/data_tables/rna.int21.dg +2329 -0
  192. eFISHent/data_tables/rna.int21.dh +2329 -0
  193. eFISHent/data_tables/rna.int22.dg +967 -0
  194. eFISHent/data_tables/rna.int22.dh +967 -0
  195. eFISHent/data_tables/rna.loop.dg +34 -0
  196. eFISHent/data_tables/rna.loop.dh +34 -0
  197. eFISHent/data_tables/rna.miscloop.dg +66 -0
  198. eFISHent/data_tables/rna.miscloop.dh +66 -0
  199. eFISHent/data_tables/rna.param_map.dg +294 -0
  200. eFISHent/data_tables/rna.specification.dat +58 -0
  201. eFISHent/data_tables/rna.stack.dg +102 -0
  202. eFISHent/data_tables/rna.stack.dh +102 -0
  203. eFISHent/data_tables/rna.tloop.dg +19 -0
  204. eFISHent/data_tables/rna.tloop.dh +18 -0
  205. eFISHent/data_tables/rna.triloop.dg +4 -0
  206. eFISHent/data_tables/rna.triloop.dh +4 -0
  207. eFISHent/data_tables/rna.tstack.dg +102 -0
  208. eFISHent/data_tables/rna.tstack.dh +102 -0
  209. eFISHent/data_tables/rna.tstackcoax.dg +102 -0
  210. eFISHent/data_tables/rna.tstackcoax.dh +102 -0
  211. eFISHent/data_tables/rna.tstackh.dg +102 -0
  212. eFISHent/data_tables/rna.tstackh.dh +102 -0
  213. eFISHent/data_tables/rna.tstacki.dg +102 -0
  214. eFISHent/data_tables/rna.tstacki.dh +102 -0
  215. eFISHent/data_tables/rna.tstacki1n.dg +102 -0
  216. eFISHent/data_tables/rna.tstacki1n.dh +102 -0
  217. eFISHent/data_tables/rna.tstacki23.dg +102 -0
  218. eFISHent/data_tables/rna.tstacki23.dh +102 -0
  219. eFISHent/data_tables/rna.tstackm.dg +102 -0
  220. eFISHent/data_tables/rna.tstackm.dh +102 -0
  221. eFISHent/data_tables/stack.dat +75 -0
  222. eFISHent/data_tables/stack.dh +75 -0
  223. eFISHent/data_tables/stackdr.dat +80 -0
  224. eFISHent/data_tables/stackdr.dh +75 -0
  225. eFISHent/data_tables/tloop.dat +19 -0
  226. eFISHent/data_tables/tloop.dh +18 -0
  227. eFISHent/data_tables/triloop.dat +4 -0
  228. eFISHent/data_tables/triloop.dh +4 -0
  229. eFISHent/data_tables/tstack.dat +76 -0
  230. eFISHent/data_tables/tstack.dh +76 -0
  231. eFISHent/data_tables/tstackcoax.dat +76 -0
  232. eFISHent/data_tables/tstackcoax.dh +76 -0
  233. eFISHent/data_tables/tstackh.dat +76 -0
  234. eFISHent/data_tables/tstackh.dh +76 -0
  235. eFISHent/data_tables/tstacki.dat +76 -0
  236. eFISHent/data_tables/tstacki.dh +76 -0
  237. eFISHent/data_tables/tstacki1n.dat +76 -0
  238. eFISHent/data_tables/tstacki1n.dh +76 -0
  239. eFISHent/data_tables/tstacki23.dat +76 -0
  240. eFISHent/data_tables/tstacki23.dh +76 -0
  241. eFISHent/data_tables/tstackm.dat +76 -0
  242. eFISHent/data_tables/tstackm.dh +76 -0
  243. eFISHent/generate_probes.py +65 -0
  244. eFISHent/indexing.py +99 -0
  245. eFISHent/kmers.py +122 -0
  246. eFISHent/optimization.py +249 -0
  247. eFISHent/prepare_sequence.py +147 -0
  248. eFISHent/secondary_structure.py +83 -0
  249. eFISHent/util.py +159 -0
  250. eFISHent-0.0.3.dist-info/METADATA +41 -0
  251. eFISHent-0.0.3.dist-info/RECORD +267 -0
  252. eFISHent-0.0.3.dist-info/WHEEL +5 -0
  253. eFISHent-0.0.3.dist-info/entry_points.txt +3 -0
  254. eFISHent-0.0.3.dist-info/top_level.txt +2 -0
  255. tests/__init__.py +0 -0
  256. tests/conftest.py +9 -0
  257. tests/test_integration.py +211 -0
  258. tests/test_unit_alignment.py +110 -0
  259. tests/test_unit_basic_filtering.py +85 -0
  260. tests/test_unit_cleanup.py +77 -0
  261. tests/test_unit_generate_probes.py +43 -0
  262. tests/test_unit_indexing.py +23 -0
  263. tests/test_unit_kmers.py +25 -0
  264. tests/test_unit_optimization.py +95 -0
  265. tests/test_unit_prepare_sequence.py +103 -0
  266. tests/test_unit_secondary_structure.py +18 -0
  267. tests/test_unit_util.py +132 -0
@@ -0,0 +1,106 @@
1
+ """Filter probes based on simple criteria."""
2
+
3
+ from typing import List
4
+ import logging
5
+ import multiprocessing
6
+ import os
7
+
8
+ import Bio.Seq
9
+ import Bio.SeqIO
10
+ import Bio.SeqRecord
11
+ import Bio.SeqUtils
12
+ import Bio.SeqUtils.MeltingTemp
13
+ import luigi
14
+
15
+ from . import util
16
+ from .config import GeneralConfig
17
+ from .config import ProbeConfig
18
+ from .generate_probes import GenerateAllProbes
19
+
20
+
21
+ def get_melting_temp(
22
+ sequence: Bio.Seq.Seq, na_concentration: float, formamide_concentration: float
23
+ ) -> float:
24
+ """Get melting temperature of candidate."""
25
+ tm_raw = Bio.SeqUtils.MeltingTemp.Tm_NN(sequence, Na=na_concentration)
26
+ melting_temp = Bio.SeqUtils.MeltingTemp.chem_correction(
27
+ tm_raw, fmd=formamide_concentration
28
+ )
29
+ return float(melting_temp)
30
+
31
+
32
+ def get_gc_content(sequence: Bio.Seq.Seq) -> float:
33
+ """Get GC content of candidate."""
34
+ return float(Bio.SeqUtils.GC(sequence))
35
+
36
+
37
+ def get_g_quadruplet_count(sequence: Bio.Seq.Seq) -> int:
38
+ """Get number of G quadruplets in candidate."""
39
+ return int(sequence.count("GGGG"))
40
+
41
+
42
+ class BasicFiltering(luigi.Task):
43
+ """Initial probe filtering based on melting temperature and GC content."""
44
+
45
+ logger = logging.getLogger("custom-logger")
46
+
47
+ def requires(self):
48
+ return GenerateAllProbes()
49
+
50
+ def output(self):
51
+ return luigi.LocalTarget(
52
+ os.path.join(util.get_output_dir(), f"{util.get_gene_name()}_basic.fasta")
53
+ )
54
+
55
+ def is_candidate_valid(
56
+ self, candidate: Bio.SeqRecord.SeqRecord, config: luigi.Config
57
+ ) -> bool:
58
+ """Check if candidate matches basic criteria."""
59
+ if config.min_tm > config.max_tm:
60
+ raise ValueError("Max TM has to be greater or equal the min TM!")
61
+ if config.min_gc > config.max_gc:
62
+ raise ValueError("Max GC has to be greater or equal the min GC!")
63
+
64
+ sequence = candidate.seq
65
+ gc_content = get_gc_content(sequence)
66
+ g_quadruplets = get_g_quadruplet_count(sequence)
67
+ melting_temp = get_melting_temp(
68
+ sequence, config.na_concentration, config.formamide_concentration
69
+ )
70
+
71
+ if (
72
+ config.min_gc <= gc_content <= config.max_gc
73
+ and config.min_tm <= melting_temp <= config.max_tm
74
+ and g_quadruplets == 0
75
+ ):
76
+ return True
77
+ return False
78
+
79
+ def _is_candidate_valid(self, candidate: Bio.SeqRecord.SeqRecord) -> bool:
80
+ """Counteracting the really weird multiprocess class behavior."""
81
+ return self.is_candidate_valid(candidate, self.config)
82
+
83
+ def screen_sequences(
84
+ self, sequences: List[Bio.SeqRecord.SeqRecord]
85
+ ) -> List[Bio.SeqRecord.SeqRecord]:
86
+ """Filter candiates not matching criteria."""
87
+ if len(sequences) > 1000:
88
+ self.config = ProbeConfig()
89
+ with multiprocessing.Pool(GeneralConfig().threads) as pool:
90
+ valid = pool.map(self._is_candidate_valid, sequences)
91
+ else:
92
+ valid = [
93
+ self.is_candidate_valid(candidate, ProbeConfig())
94
+ for candidate in sequences
95
+ ]
96
+
97
+ candidates = [seq for seq, valid in zip(sequences, valid) if valid]
98
+ return candidates
99
+
100
+ def run(self):
101
+ sequences = list(Bio.SeqIO.parse(self.input().path, "fasta"))
102
+ candidates = self.screen_sequences(sequences)
103
+ util.log_and_check_candidates(
104
+ self.logger, "BasicFiltering", len(candidates), len(sequences)
105
+ )
106
+ Bio.SeqIO.write(candidates, self.output().path, format="fasta")
eFISHent/cleanup.py ADDED
@@ -0,0 +1,163 @@
1
+ """Clean up the output directory.
2
+
3
+ Remove the files that are not needed and prettify the kept output.
4
+ """
5
+
6
+ from typing import List, Optional
7
+ import glob
8
+ import logging
9
+ import os
10
+ import sys
11
+
12
+ import luigi
13
+ import pandas as pd
14
+ import Bio.SeqIO
15
+
16
+ from . import util
17
+ from .alignment import AlignProbeCandidates
18
+ from .basic_filtering import get_gc_content
19
+ from .basic_filtering import get_melting_temp
20
+ from .config import ProbeConfig
21
+ from .config import RunConfig
22
+ from .config import SequenceConfig
23
+ from .kmers import BuildJellyfishIndex
24
+ from .kmers import get_max_kmer_count
25
+ from .optimization import OptimizeProbeCoverage
26
+ from .secondary_structure import get_free_energy
27
+
28
+
29
+ class CleanUpOutput(luigi.Task):
30
+ """Clean up the output files and remove the intermediaries that are not needed."""
31
+
32
+ logger = logging.getLogger("custom-logger")
33
+
34
+ def __init__(self, *args, **kwargs):
35
+ super().__init__(*args, **kwargs)
36
+ self.is_blast_required = (
37
+ ProbeConfig().encode_count_table and SequenceConfig().is_endogenous
38
+ )
39
+
40
+ def requires(self):
41
+ tasks = {
42
+ "optimize": OptimizeProbeCoverage(),
43
+ "jellyfish": BuildJellyfishIndex(),
44
+ }
45
+ if self.is_blast_required:
46
+ tasks["alignment"] = AlignProbeCandidates()
47
+ return tasks
48
+
49
+ def output(self):
50
+ return {
51
+ name: luigi.LocalTarget(os.path.join(util.get_output_dir(), filename))
52
+ for name, filename in [
53
+ ("fasta", f"{util.get_gene_name()}.fasta"),
54
+ ("table", f"{util.get_gene_name()}.csv"),
55
+ ("config", f"{util.get_gene_name()}.txt"),
56
+ ]
57
+ }
58
+
59
+ def prettify_table(
60
+ self,
61
+ sequences: List[Bio.SeqRecord.SeqRecord],
62
+ basename: str,
63
+ jellyfish_path: str,
64
+ alignment_path: Optional[str] = None,
65
+ config: luigi.Config = ProbeConfig,
66
+ ) -> pd.DataFrame:
67
+ """Create table with probe information."""
68
+ # Create basic table with probe start/end positions and name
69
+ df = util.create_data_table(sequences)
70
+ sequences = sorted(sequences, key=lambda x: int(x.id.split("-")[-1]))
71
+
72
+ # Add data columns
73
+ df["GC"] = [round(get_gc_content(seq.seq), 2) for seq in sequences]
74
+ df["TM"] = [
75
+ round(
76
+ get_melting_temp(
77
+ seq.seq, config().na_concentration, config().formamide_concentration
78
+ ),
79
+ 2,
80
+ )
81
+ for seq in sequences
82
+ ]
83
+ df["deltaG"] = [get_free_energy(seq) for seq in sequences]
84
+ df["kmers"] = [get_max_kmer_count(seq, jellyfish_path) for seq in sequences]
85
+ if alignment_path is not None:
86
+ df_counts = pd.read_csv(alignment_path)
87
+ df_counts = df_counts.groupby("qname", as_index=False).size()
88
+ df["count"] = pd.merge(
89
+ df, df_counts, how="left", left_on="name", right_on="qname"
90
+ )["size"].fillna(0)
91
+
92
+ # Create new/clean names
93
+ df["name"] = [f"{basename}-{idx + 1}" for idx in df.index]
94
+ return df
95
+
96
+ def prettify_sequences(self, df: pd.DataFrame) -> List[Bio.SeqRecord.SeqRecord]:
97
+ """Clean up sequence id's and descriptions."""
98
+ return [
99
+ Bio.SeqRecord.SeqRecord(
100
+ Bio.Seq.Seq(seq), id=name, name=name, description=""
101
+ )
102
+ for seq, name in zip(df["sequence"], df["name"])
103
+ ]
104
+
105
+ def prettify_section(self, section: str) -> str:
106
+ """Prettify a single configuration section."""
107
+ pretty = [f"{section.replace('Config', '')} configuration:"]
108
+ for key, value in self.config[section].items():
109
+ if value and value != '""':
110
+ pretty.append(f" {key.replace('_', '-')}: {value}")
111
+ return "\n".join(pretty)
112
+
113
+ def prettify_configuration(self) -> str:
114
+ """Create a pretty file with all set/default configuration."""
115
+ self.config = luigi.configuration.get_config()
116
+ pretty = [self.prettify_section(section) for section in self.config.sections()]
117
+ command = " ".join(sys.argv[1:])
118
+ pretty.append(f"Command:\n efishent {command}")
119
+ return "\n\n".join(pretty)
120
+
121
+ def remove_intermediates(self) -> None:
122
+ """Remove intermediary files no longer used."""
123
+ intermediary_files = glob.glob(
124
+ os.path.join(util.get_output_dir(), f"{util.get_gene_name()}_*")
125
+ )
126
+ self.logger.debug(f"Found file intermediates - {intermediary_files}")
127
+ try:
128
+ intermediary_files.remove(
129
+ os.path.join(
130
+ util.get_output_dir(), f"{util.get_gene_name()}_entrez.fasta"
131
+ )
132
+ )
133
+ except ValueError:
134
+ self.logger.debug("Didn't find entrez file...")
135
+ for filename in intermediary_files:
136
+ self.logger.debug(f"Removing {filename}")
137
+ os.remove(filename)
138
+
139
+ def run(self):
140
+ sequences = list(
141
+ Bio.SeqIO.parse(self.input()["optimize"]["probes"].path, "fasta")
142
+ )
143
+ df = self.prettify_table(
144
+ sequences,
145
+ basename=util.get_gene_name(hashed=False),
146
+ jellyfish_path=self.input()["jellyfish"].path,
147
+ alignment_path=self.input()["alignment"]["table"].path
148
+ if self.is_blast_required
149
+ else None,
150
+ )
151
+ sequences = self.prettify_sequences(df)
152
+ config = self.prettify_configuration()
153
+
154
+ util.log_and_check_candidates(self.logger, "CleanUpOutput", len(sequences))
155
+ df.to_csv(self.output()["table"].path, index=False)
156
+ Bio.SeqIO.write(sequences, self.output()["fasta"].path, format="fasta")
157
+ with open(self.output()["config"].path, "w") as f:
158
+ f.write(config)
159
+ self.logger.info(f'Saving all files using hash "{util.get_gene_name()}"')
160
+
161
+ # Files to be deleted
162
+ if not RunConfig().save_intermediates:
163
+ self.remove_intermediates()
eFISHent/cli.py ADDED
@@ -0,0 +1,214 @@
1
+ """Command line interface."""
2
+
3
+ from pathlib import Path
4
+ from typing import Any, List
5
+ import argparse
6
+ import configparser
7
+ import logging
8
+ import os
9
+ import sys
10
+ import tempfile
11
+
12
+ import luigi
13
+
14
+ # from .indexing import BuildBlastDatabase
15
+ from .analyze import AnalyzeProbeset
16
+ from .cleanup import CleanUpOutput
17
+ from .constants import CLI_SHORTFORM
18
+ from .constants import CONFIG_CLASSES
19
+ from .indexing import BuildBowtieIndex
20
+ from .kmers import BuildJellyfishIndex
21
+ from .util import UniCode
22
+
23
+
24
+ GROUP_DESCRIPTIONS = {
25
+ f"{UniCode.blue} General": "General configuration that will be used for all tasks.",
26
+ f"{UniCode.green} Run": "Options that change the behavior of the workflow.",
27
+ f"{UniCode.red} Sequence": "Details about the sequences the probe design will be performed on.",
28
+ f"{UniCode.magenta} Probe": "Probe filtering and design options.",
29
+ }
30
+ REQUIRED_PARAMS = ["reference_genome"]
31
+
32
+
33
+ def string_to_bool(value) -> bool:
34
+ """Workaround for using typed boolean values as arguments."""
35
+ if isinstance(value, bool):
36
+ return value
37
+ if value.lower() in ("yes", "true", "t", "y", "1"):
38
+ return True
39
+ elif value.lower() in ("no", "false", "f", "n", "0"):
40
+ return False
41
+
42
+ raise argparse.ArgumentTypeError("Boolean value expected.")
43
+
44
+
45
+ def get_parameter_type(param: luigi.Parameter) -> Any:
46
+ """Get the type of a parameter."""
47
+ if isinstance(param, luigi.IntParameter):
48
+ return int
49
+ elif isinstance(param, luigi.FloatParameter):
50
+ return float
51
+ elif isinstance(param, luigi.BoolParameter):
52
+ return string_to_bool
53
+ return str
54
+
55
+
56
+ def _add_utilities(parser: argparse.ArgumentParser) -> None:
57
+ """Add the utility arguments to the parser."""
58
+ utility = parser.add_argument_group(f"{UniCode.cyan}General utilities{UniCode.end}")
59
+ utility.add_argument(
60
+ "-h",
61
+ "--help",
62
+ action="help",
63
+ default=argparse.SUPPRESS,
64
+ help="Show this message.",
65
+ )
66
+ utility.add_argument(
67
+ "-V",
68
+ "--version",
69
+ action="version",
70
+ version="%(prog)s 0.0.1",
71
+ help="Show %(prog)s's version number.",
72
+ )
73
+ utility.add_argument(
74
+ "-s",
75
+ "--silent",
76
+ action="store_true",
77
+ help="Change the program output to silent hiding information on progress.",
78
+ )
79
+ utility.add_argument("--debug", action="store_true", help=argparse.SUPPRESS)
80
+
81
+
82
+ def _add_group(group: argparse._ArgumentGroup, config_class: luigi.Config) -> None:
83
+ """Add a single configuration class/group to a parser."""
84
+ for name, param in config_class().get_params():
85
+ param_type = get_parameter_type(param)
86
+ is_required = name in REQUIRED_PARAMS
87
+ default = (
88
+ "-"
89
+ if (
90
+ param._default is None
91
+ or param._default == ""
92
+ and param_type != string_to_bool
93
+ )
94
+ else param._default
95
+ )
96
+ group.add_argument(
97
+ f"-{CLI_SHORTFORM.get(name)}",
98
+ f"--{name.replace('_', '-')}",
99
+ type=param_type,
100
+ required=is_required,
101
+ default=param._default,
102
+ help=f"{param.description} "
103
+ f"[default: {default}, "
104
+ f"required: {is_required}]",
105
+ )
106
+
107
+
108
+ def _add_groups(parser: argparse.ArgumentParser) -> None:
109
+ """Add the main option groups to the parser."""
110
+ groups = [
111
+ parser.add_argument_group(
112
+ f"{name} options{UniCode.end}", description=description
113
+ )
114
+ for name, description in GROUP_DESCRIPTIONS.items()
115
+ ]
116
+
117
+ for group, config_class in zip(groups, CONFIG_CLASSES):
118
+ _add_group(group, config_class)
119
+
120
+
121
+ def _parse_args() -> argparse.Namespace:
122
+ """Parse command line arguments."""
123
+ parser = argparse.ArgumentParser(
124
+ prog="eFISHent",
125
+ description=f"{UniCode.bold}eFISHent {UniCode.fishing} {UniCode.dna} to design all your probes.{UniCode.end}",
126
+ epilog=(
127
+ 'See the online wiki at "https://github.com/BBQuercus/eFISHent/wiki" for an overview.\n'
128
+ f"We hope you enjoy using eFISHent {UniCode.party}!"
129
+ ),
130
+ add_help=False,
131
+ )
132
+ _add_groups(parser)
133
+ _add_utilities(parser)
134
+ try:
135
+ if len(sys.argv) == 1:
136
+ parser.print_help()
137
+ parser.exit(0)
138
+ except Exception as e:
139
+ print(e)
140
+ return parser.parse_args()
141
+
142
+
143
+ def create_custom_config(args: argparse.Namespace, config_file: str) -> None:
144
+ """Create a custom config file."""
145
+ config = configparser.ConfigParser()
146
+ config_path = Path(__file__).resolve().parent.joinpath("luigi.cfg").as_posix()
147
+ config.read(config_path)
148
+
149
+ for section, config_class in zip(
150
+ ["GeneralConfig", "RunConfig", "SequenceConfig", "ProbeConfig"], CONFIG_CLASSES
151
+ ):
152
+ for name in config_class().get_param_names():
153
+ value = vars(args).get(name)
154
+ config.set(section, name, str(value))
155
+ if name == "threads":
156
+ threads = min(vars(args).get(name), os.cpu_count()) # type: ignore
157
+ config.set(section, name, str(threads))
158
+
159
+ with open(config_file, "w") as f:
160
+ config.write(f)
161
+
162
+
163
+ def set_logging_level(silent: bool, debug: bool) -> logging.Logger:
164
+ """Set the logging level of luigi and custom logger."""
165
+ log_format = "%(asctime)s %(levelname)-4s - %(message)s"
166
+ luigi_level = "WARNING"
167
+ logfile = None
168
+
169
+ if debug:
170
+ log_format = (
171
+ "%(asctime)s %(levelname)-4s [%(name)s] "
172
+ "%(filename)s %(funcName)s %(lineno)d / %(thread)d - %(message)s"
173
+ )
174
+ luigi_level = "DEBUG"
175
+ custom_level = logging.DEBUG
176
+ logfile = "efishent.log"
177
+ elif silent:
178
+ custom_level = logging.WARNING
179
+ else:
180
+ custom_level = logging.INFO
181
+
182
+ logging.basicConfig(filename=logfile, format=log_format, force=True) # type: ignore
183
+ logging.getLogger("luigi").setLevel(luigi_level)
184
+ logging.getLogger("luigi-interface").setLevel(luigi_level)
185
+ luigi.interface.core.log_level = luigi_level
186
+
187
+ logger = logging.getLogger("custom-logger")
188
+ logger.setLevel(custom_level)
189
+ return logger
190
+
191
+
192
+ def main():
193
+ """Run eFISHent tasks."""
194
+ args = _parse_args()
195
+ logger = set_logging_level(args.silent, args.debug)
196
+ logger.info(f"{UniCode.fishing} eFISHent has started running...")
197
+
198
+ with tempfile.TemporaryDirectory() as tmp_dir:
199
+ config_file = os.path.join(tmp_dir, "luigi.cfg")
200
+ create_custom_config(args, config_file)
201
+ luigi.configuration.add_config_path(config_file)
202
+
203
+ tasks: List[luigi.Task] = []
204
+ if args.build_indices:
205
+ tasks = [BuildJellyfishIndex(), BuildBowtieIndex()]
206
+ elif args.analyze_probeset:
207
+ tasks = [AnalyzeProbeset()]
208
+ else:
209
+ tasks = [CleanUpOutput()]
210
+
211
+ luigi.build(tasks, local_scheduler=True)
212
+
213
+ if tasks[-1].complete():
214
+ logger.info(f"{UniCode.party} eFISHent has finished running!")
eFISHent/config.py ADDED
@@ -0,0 +1,181 @@
1
+ """Luigi configurations passed as command line arguments."""
2
+
3
+ import luigi
4
+
5
+
6
+ class GeneralConfig(luigi.Config):
7
+ reference_genome = luigi.Parameter(
8
+ description="Path to reference genome fasta/fa file.",
9
+ significant=True,
10
+ default="",
11
+ )
12
+ reference_annotation = luigi.Parameter(
13
+ description="Path to reference genome annotation (gene transfer format / gtf) file.",
14
+ default="",
15
+ )
16
+ threads = luigi.IntParameter(description="Number of threads to launch.", default=2)
17
+ output_dir = luigi.Parameter(
18
+ description=(
19
+ "Path to output directory. "
20
+ "If not specified, will use the current directory."
21
+ ),
22
+ default="",
23
+ )
24
+
25
+
26
+ class RunConfig(luigi.Config):
27
+ build_indices = luigi.BoolParameter(
28
+ description="Build indices for reference genome only.", default=False
29
+ )
30
+ analyze_probeset = luigi.Parameter(
31
+ description=(
32
+ "Path to the workflow's output fasta file. "
33
+ "Will analyze the probe set in more detail and create a PDF. "
34
+ "Note: reference genome, target gene of interest, "
35
+ "and (optional) count table will still have to be provided!"
36
+ ),
37
+ default="",
38
+ )
39
+ save_intermediates = luigi.BoolParameter(
40
+ description="Save intermediate files?", default=False
41
+ )
42
+ optimization_method = luigi.Parameter(
43
+ description=(
44
+ "Optimization method to use. "
45
+ "Greedy will procedurally select the next longest probe. "
46
+ "Optimal will optimize probes for maximum gene coverage - "
47
+ "slow if many overlaps are present (typically with non-stringent parameter settings). "
48
+ "[options: optimal, greedy]."
49
+ ),
50
+ default="greedy",
51
+ )
52
+ optimization_time_limit = luigi.IntParameter(
53
+ description=(
54
+ "Time limit for optimization in seconds. "
55
+ "Only used if optimization method is set to 'optimal'."
56
+ ),
57
+ default=60,
58
+ )
59
+
60
+
61
+ class SequenceConfig(luigi.Config):
62
+ sequence_file = luigi.Parameter(
63
+ description=(
64
+ "Path to the gene's sequence file. "
65
+ "Use this if the sequence can't be easily downloaded or if only certain regions should be targeted."
66
+ ),
67
+ default="",
68
+ )
69
+ ensembl_id = luigi.Parameter(
70
+ description=(
71
+ "Ensembl ID of the gene of interest. "
72
+ "Can be used instead of gene and organism name to download the gene of interest. "
73
+ "Used to filter out the gene of interest from FPKM based filtering - "
74
+ "will do an automated blast-based filtering if not passed."
75
+ ),
76
+ default="",
77
+ )
78
+ gene_name = luigi.Parameter(description="Gene name.", default="")
79
+ organism_name = luigi.Parameter(
80
+ description=(
81
+ "Latin name of the organism. "
82
+ "Can be passed together with `ensembl_id` to narrow search."
83
+ ),
84
+ default="",
85
+ )
86
+ is_plus_strand = luigi.BoolParameter(
87
+ description=(
88
+ "Is the probe targeting the plus strand? "
89
+ "Note: Entrez downloads will download the gene 5'-3'. "
90
+ "Specifying the strand is therefore not required."
91
+ ),
92
+ default=True,
93
+ )
94
+ is_endogenous = luigi.BoolParameter(
95
+ description="Is the probe endogenous?", default=True
96
+ )
97
+
98
+
99
+ class ProbeConfig(luigi.Config):
100
+ min_length = luigi.IntParameter(
101
+ description="Minimum probe length in nucleotides.", default=21
102
+ )
103
+ max_length = luigi.IntParameter(
104
+ description="Maximum probe length in nucleotides.", default=25
105
+ )
106
+ spacing = luigi.IntParameter(
107
+ description="Minimum distance in nucleotides between probes.", default=2
108
+ )
109
+ min_tm = luigi.FloatParameter(
110
+ description=(
111
+ "Minimum melting temperature in degrees C. "
112
+ "Formamide and Na concentration will affect melting temperature!"
113
+ ),
114
+ default=40.0,
115
+ )
116
+ max_tm = luigi.FloatParameter(
117
+ description="Maximum melting temperature in degrees C (see minimum).",
118
+ default=60.0,
119
+ )
120
+ min_gc = luigi.FloatParameter(
121
+ description="Minimum GC content in percentage.", default=20.0
122
+ )
123
+ max_gc = luigi.FloatParameter(
124
+ description="Maximum GC content in percentage.", default=80.0
125
+ )
126
+ formamide_concentration = luigi.FloatParameter(
127
+ description="Formamide concentration as a percentage of the total buffer.",
128
+ default=10.0,
129
+ )
130
+ na_concentration = luigi.FloatParameter(
131
+ description="Na concentration in mM.", default=330.0
132
+ )
133
+ max_off_targets = luigi.IntParameter(
134
+ description=(
135
+ "Maximum number of off-targets binding anywhere BUT "
136
+ "the gene of interest in the genome."
137
+ ),
138
+ default=0,
139
+ )
140
+ encode_count_table = luigi.Parameter(
141
+ description=(
142
+ "Path to the ENCODE RNAseq count table provided as TSV, CSV, or TXT format. "
143
+ "The first two columns have to be `gene_id` and `normalized_value` respectively. "
144
+ "Specific column names are not required. "
145
+ "`gene_id` have to be ensemble IDs of the respective genes. "
146
+ "`normalized_value` have to be normalized count values (RPKM, FPKM, TPM, DeSeq2-norm, etc). "
147
+ ),
148
+ default="",
149
+ )
150
+ max_expression_percentage = luigi.FloatParameter(
151
+ description=(
152
+ "Remove any off-target genes expressed more highly than "
153
+ "`max-expression-percentage` percentage of all genes. "
154
+ "Genes to remove are based on ENCODE RNAseq count table provided in `encode-count-table`. "
155
+ "Only used if `encode-count-table` and `reference-annotation` are provided."
156
+ ),
157
+ default=50.0,
158
+ )
159
+ kmer_length = luigi.IntParameter(
160
+ description="Length of k-mer used to filter probe sequences.", default=15
161
+ )
162
+ max_kmers = luigi.IntParameter(
163
+ description="Highest count of sub-k-mers found in reference genome.", default=5
164
+ )
165
+ max_deltag = luigi.FloatParameter(
166
+ description=(
167
+ "Maximum predicted deltaG in kcal/mol. "
168
+ "Note: deltaGs range from 0 (no secondary structures) to increasingly negative values!"
169
+ ),
170
+ default=-10.0,
171
+ )
172
+ sequence_similarity = luigi.IntParameter(
173
+ description=(
174
+ "Maximum percentage probes can be similar. "
175
+ "Applied to their complements and reverse complements only. "
176
+ "Ensures probes don't falsely bind to eachother. "
177
+ "Setting any value above 0 could add multiple minute of runtime! "
178
+ "The lower the value (above 0) the fewer potential probes."
179
+ ),
180
+ default=0,
181
+ )