eFISHent 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (267) hide show
  1. eFISHent/__init__.py +23 -0
  2. eFISHent/__main__.py +6 -0
  3. eFISHent/alignment.py +336 -0
  4. eFISHent/analyze.py +246 -0
  5. eFISHent/basic_filtering.py +106 -0
  6. eFISHent/cleanup.py +163 -0
  7. eFISHent/cli.py +214 -0
  8. eFISHent/config.py +181 -0
  9. eFISHent/constants.py +95 -0
  10. eFISHent/data_tables/b-test.coaxial.dg +111 -0
  11. eFISHent/data_tables/b-test.coaxial.dh +111 -0
  12. eFISHent/data_tables/b-test.coaxstack.dg +349 -0
  13. eFISHent/data_tables/b-test.coaxstack.dh +349 -0
  14. eFISHent/data_tables/b-test.dangle.dg +499 -0
  15. eFISHent/data_tables/b-test.dangle.dh +499 -0
  16. eFISHent/data_tables/b-test.dynalignmiscloop.dg +62 -0
  17. eFISHent/data_tables/b-test.hexaloop.dg +26 -0
  18. eFISHent/data_tables/b-test.hexaloop.dh +26 -0
  19. eFISHent/data_tables/b-test.int11.dg +1047 -0
  20. eFISHent/data_tables/b-test.int11.dh +895 -0
  21. eFISHent/data_tables/b-test.int21.dg +5897 -0
  22. eFISHent/data_tables/b-test.int21.dh +3583 -0
  23. eFISHent/data_tables/b-test.int22.dg +2239 -0
  24. eFISHent/data_tables/b-test.int22.dh +2239 -0
  25. eFISHent/data_tables/b-test.loop.dg +34 -0
  26. eFISHent/data_tables/b-test.miscloop.dg +66 -0
  27. eFISHent/data_tables/b-test.miscloop.dh +66 -0
  28. eFISHent/data_tables/b-test.specification.dat +64 -0
  29. eFISHent/data_tables/b-test.stack.dg +111 -0
  30. eFISHent/data_tables/b-test.stack.dh +111 -0
  31. eFISHent/data_tables/b-test.tloop.dg +34 -0
  32. eFISHent/data_tables/b-test.tloop.dh +34 -0
  33. eFISHent/data_tables/b-test.triloop.dg +8 -0
  34. eFISHent/data_tables/b-test.triloop.dh +8 -0
  35. eFISHent/data_tables/b-test.tstack.dg +111 -0
  36. eFISHent/data_tables/b-test.tstack.dh +111 -0
  37. eFISHent/data_tables/b-test.tstackcoax.dg +111 -0
  38. eFISHent/data_tables/b-test.tstackcoax.dh +111 -0
  39. eFISHent/data_tables/b-test.tstackh.dg +111 -0
  40. eFISHent/data_tables/b-test.tstackh.dh +111 -0
  41. eFISHent/data_tables/b-test.tstacki.dg +111 -0
  42. eFISHent/data_tables/b-test.tstacki.dh +111 -0
  43. eFISHent/data_tables/b-test.tstacki1n.dg +111 -0
  44. eFISHent/data_tables/b-test.tstacki1n.dh +111 -0
  45. eFISHent/data_tables/b-test.tstacki23.dg +111 -0
  46. eFISHent/data_tables/b-test.tstacki23.dh +111 -0
  47. eFISHent/data_tables/b-test.tstackm.dg +111 -0
  48. eFISHent/data_tables/b-test.tstackm.dh +111 -0
  49. eFISHent/data_tables/coaxial.dat +75 -0
  50. eFISHent/data_tables/coaxial.dh +75 -0
  51. eFISHent/data_tables/coaxstack.dat +75 -0
  52. eFISHent/data_tables/coaxstack.dh +75 -0
  53. eFISHent/data_tables/dangle.dat +89 -0
  54. eFISHent/data_tables/dangle.dh +89 -0
  55. eFISHent/data_tables/design.DNA.Helices.dat +2000 -0
  56. eFISHent/data_tables/design.DNA.Loops.dat +2000 -0
  57. eFISHent/data_tables/design.RNA.Helices.dat +2000 -0
  58. eFISHent/data_tables/design.RNA.Loops.dat +2075 -0
  59. eFISHent/data_tables/dna.coaxial.dg +102 -0
  60. eFISHent/data_tables/dna.coaxial.dh +102 -0
  61. eFISHent/data_tables/dna.coaxstack.dg +244 -0
  62. eFISHent/data_tables/dna.coaxstack.dh +244 -0
  63. eFISHent/data_tables/dna.dangle.dg +363 -0
  64. eFISHent/data_tables/dna.dangle.dh +363 -0
  65. eFISHent/data_tables/dna.dynalignmiscloop.dg +54 -0
  66. eFISHent/data_tables/dna.dynalignmiscloop.dh +67 -0
  67. eFISHent/data_tables/dna.hexaloop.dg +2 -0
  68. eFISHent/data_tables/dna.hexaloop.dh +2 -0
  69. eFISHent/data_tables/dna.int11.dg +601 -0
  70. eFISHent/data_tables/dna.int11.dh +601 -0
  71. eFISHent/data_tables/dna.int21.dg +2329 -0
  72. eFISHent/data_tables/dna.int21.dh +2329 -0
  73. eFISHent/data_tables/dna.int22.dg +967 -0
  74. eFISHent/data_tables/dna.int22.dh +967 -0
  75. eFISHent/data_tables/dna.loop.dg +34 -0
  76. eFISHent/data_tables/dna.loop.dh +34 -0
  77. eFISHent/data_tables/dna.miscloop.dg +67 -0
  78. eFISHent/data_tables/dna.miscloop.dh +67 -0
  79. eFISHent/data_tables/dna.specification.dat +58 -0
  80. eFISHent/data_tables/dna.stack.dg +102 -0
  81. eFISHent/data_tables/dna.stack.dh +102 -0
  82. eFISHent/data_tables/dna.tloop.dg +9 -0
  83. eFISHent/data_tables/dna.tloop.dh +8 -0
  84. eFISHent/data_tables/dna.triloop.dg +5 -0
  85. eFISHent/data_tables/dna.triloop.dh +5 -0
  86. eFISHent/data_tables/dna.tstack.dg +102 -0
  87. eFISHent/data_tables/dna.tstack.dh +102 -0
  88. eFISHent/data_tables/dna.tstackcoax.dg +102 -0
  89. eFISHent/data_tables/dna.tstackcoax.dh +102 -0
  90. eFISHent/data_tables/dna.tstackh.dg +102 -0
  91. eFISHent/data_tables/dna.tstackh.dh +102 -0
  92. eFISHent/data_tables/dna.tstacki.dg +102 -0
  93. eFISHent/data_tables/dna.tstacki.dh +102 -0
  94. eFISHent/data_tables/dna.tstacki1n.dg +102 -0
  95. eFISHent/data_tables/dna.tstacki1n.dh +102 -0
  96. eFISHent/data_tables/dna.tstacki23.dg +102 -0
  97. eFISHent/data_tables/dna.tstacki23.dh +102 -0
  98. eFISHent/data_tables/dna.tstackm.dg +102 -0
  99. eFISHent/data_tables/dna.tstackm.dh +102 -0
  100. eFISHent/data_tables/dnacoaxial.dat +76 -0
  101. eFISHent/data_tables/dnacoaxial.dh +76 -0
  102. eFISHent/data_tables/dnacoaxstack.dat +75 -0
  103. eFISHent/data_tables/dnacoaxstack.dh +76 -0
  104. eFISHent/data_tables/dnadangle.dat +89 -0
  105. eFISHent/data_tables/dnadangle.dh +89 -0
  106. eFISHent/data_tables/dnadynalignmiscloop.dat +54 -0
  107. eFISHent/data_tables/dnadynalignmiscloop.dh +67 -0
  108. eFISHent/data_tables/dnahexaloop.dat +2 -0
  109. eFISHent/data_tables/dnahexaloop.dh +2 -0
  110. eFISHent/data_tables/dnaint11.dat +112 -0
  111. eFISHent/data_tables/dnaint11.dh +117 -0
  112. eFISHent/data_tables/dnaint21.dat +356 -0
  113. eFISHent/data_tables/dnaint21.dh +355 -0
  114. eFISHent/data_tables/dnaint22.dat +967 -0
  115. eFISHent/data_tables/dnaint22.dh +967 -0
  116. eFISHent/data_tables/dnaloop.dat +34 -0
  117. eFISHent/data_tables/dnaloop.dh +34 -0
  118. eFISHent/data_tables/dnamiscloop.dat +63 -0
  119. eFISHent/data_tables/dnamiscloop.dh +63 -0
  120. eFISHent/data_tables/dnastack.dat +76 -0
  121. eFISHent/data_tables/dnastack.dh +76 -0
  122. eFISHent/data_tables/dnatloop.dat +116 -0
  123. eFISHent/data_tables/dnatloop.dh +14 -0
  124. eFISHent/data_tables/dnatriloop.dat +19 -0
  125. eFISHent/data_tables/dnatriloop.dh +5 -0
  126. eFISHent/data_tables/dnatstack.dat +76 -0
  127. eFISHent/data_tables/dnatstack.dh +76 -0
  128. eFISHent/data_tables/dnatstackcoax.dat +76 -0
  129. eFISHent/data_tables/dnatstackcoax.dh +76 -0
  130. eFISHent/data_tables/dnatstackh.dat +76 -0
  131. eFISHent/data_tables/dnatstackh.dh +76 -0
  132. eFISHent/data_tables/dnatstacki.dat +76 -0
  133. eFISHent/data_tables/dnatstacki.dh +76 -0
  134. eFISHent/data_tables/dnatstacki1n.dat +76 -0
  135. eFISHent/data_tables/dnatstacki1n.dh +76 -0
  136. eFISHent/data_tables/dnatstacki23.dat +76 -0
  137. eFISHent/data_tables/dnatstacki23.dh +76 -0
  138. eFISHent/data_tables/dnatstackm.dat +76 -0
  139. eFISHent/data_tables/dnatstackm.dh +76 -0
  140. eFISHent/data_tables/dynalignmiscloop.dat +62 -0
  141. eFISHent/data_tables/fam_hmm_pars.dat +1 -0
  142. eFISHent/data_tables/helix.dat +5 -0
  143. eFISHent/data_tables/helixdr.dat +5 -0
  144. eFISHent/data_tables/hexaloop.dat +6 -0
  145. eFISHent/data_tables/hexaloop.dh +6 -0
  146. eFISHent/data_tables/int11.dat +113 -0
  147. eFISHent/data_tables/int11.dh +113 -0
  148. eFISHent/data_tables/int21.dat +371 -0
  149. eFISHent/data_tables/int21.dh +371 -0
  150. eFISHent/data_tables/int22-exp.dh +967 -0
  151. eFISHent/data_tables/int22.dat +967 -0
  152. eFISHent/data_tables/int22.dh +967 -0
  153. eFISHent/data_tables/loop.dat +34 -0
  154. eFISHent/data_tables/loop.dh +34 -0
  155. eFISHent/data_tables/m6A.coaxial.dg +134 -0
  156. eFISHent/data_tables/m6A.coaxstack.dg +374 -0
  157. eFISHent/data_tables/m6A.dangle.dg +210 -0
  158. eFISHent/data_tables/m6A.hexaloop.dg +6 -0
  159. eFISHent/data_tables/m6A.int11.dg +1133 -0
  160. eFISHent/data_tables/m6A.int21.dg +5440 -0
  161. eFISHent/data_tables/m6A.int22.dg +2304 -0
  162. eFISHent/data_tables/m6A.loop.dg +34 -0
  163. eFISHent/data_tables/m6A.miscloop.dg +66 -0
  164. eFISHent/data_tables/m6A.specification.dat +64 -0
  165. eFISHent/data_tables/m6A.stack.dg +134 -0
  166. eFISHent/data_tables/m6A.tloop.dg +19 -0
  167. eFISHent/data_tables/m6A.triloop.dg +4 -0
  168. eFISHent/data_tables/m6A.tstack.dg +136 -0
  169. eFISHent/data_tables/m6A.tstackcoax.dg +136 -0
  170. eFISHent/data_tables/m6A.tstackh.dg +137 -0
  171. eFISHent/data_tables/m6A.tstacki.dg +142 -0
  172. eFISHent/data_tables/m6A.tstacki1n.dg +140 -0
  173. eFISHent/data_tables/m6A.tstacki23.dg +144 -0
  174. eFISHent/data_tables/m6A.tstackm.dg +136 -0
  175. eFISHent/data_tables/miscloop.dat +62 -0
  176. eFISHent/data_tables/miscloop.dh +62 -0
  177. eFISHent/data_tables/pseudconst.dat +34 -0
  178. eFISHent/data_tables/rna.coaxial.dg +102 -0
  179. eFISHent/data_tables/rna.coaxial.dh +102 -0
  180. eFISHent/data_tables/rna.coaxstack.dg +244 -0
  181. eFISHent/data_tables/rna.coaxstack.dh +244 -0
  182. eFISHent/data_tables/rna.cov.dg +294 -0
  183. eFISHent/data_tables/rna.cov.dh +294 -0
  184. eFISHent/data_tables/rna.dangle.dg +363 -0
  185. eFISHent/data_tables/rna.dangle.dh +363 -0
  186. eFISHent/data_tables/rna.dynalignmiscloop.dg +62 -0
  187. eFISHent/data_tables/rna.hexaloop.dg +6 -0
  188. eFISHent/data_tables/rna.hexaloop.dh +6 -0
  189. eFISHent/data_tables/rna.int11.dg +601 -0
  190. eFISHent/data_tables/rna.int11.dh +601 -0
  191. eFISHent/data_tables/rna.int21.dg +2329 -0
  192. eFISHent/data_tables/rna.int21.dh +2329 -0
  193. eFISHent/data_tables/rna.int22.dg +967 -0
  194. eFISHent/data_tables/rna.int22.dh +967 -0
  195. eFISHent/data_tables/rna.loop.dg +34 -0
  196. eFISHent/data_tables/rna.loop.dh +34 -0
  197. eFISHent/data_tables/rna.miscloop.dg +66 -0
  198. eFISHent/data_tables/rna.miscloop.dh +66 -0
  199. eFISHent/data_tables/rna.param_map.dg +294 -0
  200. eFISHent/data_tables/rna.specification.dat +58 -0
  201. eFISHent/data_tables/rna.stack.dg +102 -0
  202. eFISHent/data_tables/rna.stack.dh +102 -0
  203. eFISHent/data_tables/rna.tloop.dg +19 -0
  204. eFISHent/data_tables/rna.tloop.dh +18 -0
  205. eFISHent/data_tables/rna.triloop.dg +4 -0
  206. eFISHent/data_tables/rna.triloop.dh +4 -0
  207. eFISHent/data_tables/rna.tstack.dg +102 -0
  208. eFISHent/data_tables/rna.tstack.dh +102 -0
  209. eFISHent/data_tables/rna.tstackcoax.dg +102 -0
  210. eFISHent/data_tables/rna.tstackcoax.dh +102 -0
  211. eFISHent/data_tables/rna.tstackh.dg +102 -0
  212. eFISHent/data_tables/rna.tstackh.dh +102 -0
  213. eFISHent/data_tables/rna.tstacki.dg +102 -0
  214. eFISHent/data_tables/rna.tstacki.dh +102 -0
  215. eFISHent/data_tables/rna.tstacki1n.dg +102 -0
  216. eFISHent/data_tables/rna.tstacki1n.dh +102 -0
  217. eFISHent/data_tables/rna.tstacki23.dg +102 -0
  218. eFISHent/data_tables/rna.tstacki23.dh +102 -0
  219. eFISHent/data_tables/rna.tstackm.dg +102 -0
  220. eFISHent/data_tables/rna.tstackm.dh +102 -0
  221. eFISHent/data_tables/stack.dat +75 -0
  222. eFISHent/data_tables/stack.dh +75 -0
  223. eFISHent/data_tables/stackdr.dat +80 -0
  224. eFISHent/data_tables/stackdr.dh +75 -0
  225. eFISHent/data_tables/tloop.dat +19 -0
  226. eFISHent/data_tables/tloop.dh +18 -0
  227. eFISHent/data_tables/triloop.dat +4 -0
  228. eFISHent/data_tables/triloop.dh +4 -0
  229. eFISHent/data_tables/tstack.dat +76 -0
  230. eFISHent/data_tables/tstack.dh +76 -0
  231. eFISHent/data_tables/tstackcoax.dat +76 -0
  232. eFISHent/data_tables/tstackcoax.dh +76 -0
  233. eFISHent/data_tables/tstackh.dat +76 -0
  234. eFISHent/data_tables/tstackh.dh +76 -0
  235. eFISHent/data_tables/tstacki.dat +76 -0
  236. eFISHent/data_tables/tstacki.dh +76 -0
  237. eFISHent/data_tables/tstacki1n.dat +76 -0
  238. eFISHent/data_tables/tstacki1n.dh +76 -0
  239. eFISHent/data_tables/tstacki23.dat +76 -0
  240. eFISHent/data_tables/tstacki23.dh +76 -0
  241. eFISHent/data_tables/tstackm.dat +76 -0
  242. eFISHent/data_tables/tstackm.dh +76 -0
  243. eFISHent/generate_probes.py +65 -0
  244. eFISHent/indexing.py +99 -0
  245. eFISHent/kmers.py +122 -0
  246. eFISHent/optimization.py +249 -0
  247. eFISHent/prepare_sequence.py +147 -0
  248. eFISHent/secondary_structure.py +83 -0
  249. eFISHent/util.py +159 -0
  250. eFISHent-0.0.3.dist-info/METADATA +41 -0
  251. eFISHent-0.0.3.dist-info/RECORD +267 -0
  252. eFISHent-0.0.3.dist-info/WHEEL +5 -0
  253. eFISHent-0.0.3.dist-info/entry_points.txt +3 -0
  254. eFISHent-0.0.3.dist-info/top_level.txt +2 -0
  255. tests/__init__.py +0 -0
  256. tests/conftest.py +9 -0
  257. tests/test_integration.py +211 -0
  258. tests/test_unit_alignment.py +110 -0
  259. tests/test_unit_basic_filtering.py +85 -0
  260. tests/test_unit_cleanup.py +77 -0
  261. tests/test_unit_generate_probes.py +43 -0
  262. tests/test_unit_indexing.py +23 -0
  263. tests/test_unit_kmers.py +25 -0
  264. tests/test_unit_optimization.py +95 -0
  265. tests/test_unit_prepare_sequence.py +103 -0
  266. tests/test_unit_secondary_structure.py +18 -0
  267. tests/test_unit_util.py +132 -0
eFISHent/__init__.py ADDED
@@ -0,0 +1,23 @@
1
+ """eFISHent."""
2
+
3
+ __version__ = "0.0.3"
4
+
5
+ import logging
6
+
7
+ # Silence some troublemakers
8
+ logging.getLogger("luigi-interface").setLevel(level=logging.CRITICAL)
9
+ logging.getLogger("matplotlib.font_manager").setLevel(level=logging.CRITICAL)
10
+
11
+ from . import alignment
12
+ from . import analyze
13
+ from . import basic_filtering
14
+ from . import cleanup
15
+ from . import cli
16
+ from . import config
17
+ from . import constants
18
+ from . import generate_probes
19
+ from . import kmers
20
+ from . import optimization
21
+ from . import prepare_sequence
22
+ from . import secondary_structure
23
+ from . import util
eFISHent/__main__.py ADDED
@@ -0,0 +1,6 @@
1
+ """Entrypoint module for the application."""
2
+
3
+ from .cli import main
4
+
5
+ if __name__ == "__main__":
6
+ main()
eFISHent/alignment.py ADDED
@@ -0,0 +1,336 @@
1
+ """Filter probes based on alignment score and uniqueness."""
2
+
3
+ # from typing import Any
4
+ import logging
5
+ import os
6
+ import subprocess
7
+
8
+ # import numpy as np
9
+ import Bio.SeqIO
10
+ import luigi
11
+ import pandas as pd
12
+ import pysam
13
+
14
+ # from .indexing import BuildBlastDatabase
15
+ # from .indexing import PrepareAnnotationFile
16
+ # from .prepare_sequence import PrepareSequence
17
+ from . import constants
18
+ from . import util
19
+ from .basic_filtering import BasicFiltering
20
+ from .config import GeneralConfig
21
+ from .config import ProbeConfig
22
+ from .config import SequenceConfig
23
+ from .indexing import BuildBowtieIndex
24
+
25
+
26
+ class AlignProbeCandidates(luigi.Task):
27
+ """Align all probe candidates to the reference genome and filter."""
28
+
29
+ logger = logging.getLogger("custom-logger")
30
+
31
+ def __init__(self, *args, **kwargs):
32
+ super().__init__(*args, **kwargs)
33
+ # self.is_blast_required = (
34
+ # ProbeConfig().encode_count_table and SequenceConfig().is_endogenous
35
+ # )
36
+ self.is_endogenous = SequenceConfig().is_endogenous
37
+ self.max_off_targets = ProbeConfig().max_off_targets
38
+
39
+ def requires(self):
40
+ tasks = {"probes": BasicFiltering(), "bowtie": BuildBowtieIndex()}
41
+ # if self.is_blast_required:
42
+ # tasks["blastseq"] = PrepareSequence()
43
+ # tasks["gtf"] = PrepareAnnotationFile()
44
+ # tasks["blast"] = BuildBlastDatabase()
45
+ return tasks
46
+
47
+ def output(self):
48
+ tasks = {
49
+ "fasta": luigi.LocalTarget(
50
+ os.path.join(
51
+ util.get_output_dir(), f"{util.get_gene_name()}_aligned.fasta"
52
+ )
53
+ ),
54
+ "table": luigi.LocalTarget(
55
+ os.path.join(
56
+ util.get_output_dir(), f"{util.get_gene_name()}_counts.csv"
57
+ )
58
+ ),
59
+ }
60
+ return tasks
61
+
62
+ @property
63
+ def count_table(self) -> pd.DataFrame:
64
+ """Read and verify a RNAseq normalized count table."""
65
+ extension = os.path.splitext(self.fname_count)[1].lower()
66
+ if extension not in (".tsv", ".csv", ".txt"):
67
+ raise ValueError(
68
+ "The count table must be provided as TSV, CSV, or TXT file. "
69
+ f"Only found - {self.fname_count}"
70
+ )
71
+ sep = "\t" if extension in (".tsv", ".txt") else ","
72
+ df = pd.read_csv(self.fname_count, sep=sep, index_col=0).reset_index().dropna()
73
+ if not len(df.columns) >= 2:
74
+ raise ValueError(
75
+ "The count table must contain at least 2 columns."
76
+ f"Only found - {df.columns}"
77
+ )
78
+ self.logger.debug(f"Mapping original the first two columns of - {df.columns}.")
79
+ self.logger.debug(f"Read count table with {len(df)} entries.")
80
+ self.logger.debug(f"Found dtypes - {df.dtypes}")
81
+ self.logger.debug(f"First column - {df.iloc[0]}")
82
+
83
+ df = df.iloc[:, :2]
84
+ df.columns = ["gene_id", "count"]
85
+ try:
86
+ df = df.astype({"gene_id": str, "count": float})
87
+ except ValueError:
88
+ raise ValueError(
89
+ "Could not convert count table columns to the right type. "
90
+ "Please ensure the first two colums are of type string and number/float/int, respectively. "
91
+ )
92
+
93
+ # Remove potential version numbers
94
+ df["gene_id"] = df["gene_id"].apply(lambda x: x.split(".")[0])
95
+ return df
96
+
97
+ @property
98
+ def gtf_table(self) -> pd.DataFrame:
99
+ """Parse prepared parquet-formatted gtf file."""
100
+ return pd.read_parquet(self.fname_gtf)[constants.GTF_COLUMNS]
101
+
102
+ @property
103
+ def norm_table(self) -> pd.DataFrame:
104
+ """Merge gtf and count table."""
105
+ df = pd.merge(
106
+ self.count_table, self.gtf_table, how="left", on="gene_id"
107
+ ).dropna()
108
+ return df
109
+
110
+ def align_probes(self, threads: int):
111
+ """Align probes to the reference genome.
112
+
113
+ Takes a fasta file as input (which will be converted to fastq).
114
+ Saves a sam file with alignments.
115
+ """
116
+ # Convert fasta to fastq - bowtie only allows uses fastq inputs
117
+ fname_fastq = self.fname_fasta.rstrip("a") + "q"
118
+ with open(self.fname_fasta, "r") as fasta, open(fname_fastq, "w") as fastq:
119
+ for record in Bio.SeqIO.parse(fasta, "fasta"):
120
+ record.letter_annotations["phred_quality"] = [40] * len(record)
121
+ Bio.SeqIO.write(record, fastq, format="fastq")
122
+ self.logger.debug(f"Converted fasta to fastq - {fname_fastq}")
123
+
124
+ # Actual alignment with -m to only return alignments with >m hits
125
+ alignments = str(max(self.max_off_targets + 1, 1))
126
+ endo_exo_param = [] if self.is_endogenous else ["-m", alignments]
127
+ args_bowtie = [
128
+ "bowtie",
129
+ fname_fastq,
130
+ "-x",
131
+ self.fname_genome,
132
+ "--threads",
133
+ str(threads),
134
+ *endo_exo_param,
135
+ "--sam",
136
+ self.fname_sam,
137
+ "--all",
138
+ ]
139
+ self.logger.debug(f"Running bowtie with - {' '.join(args_bowtie)}")
140
+ subprocess.check_call(
141
+ args_bowtie, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT
142
+ )
143
+
144
+ @staticmethod
145
+ def parse_raw_pysam(sam: str) -> pd.DataFrame:
146
+ """Convert string based pysam output to a DataFrame."""
147
+ # Parse tab and newline delimited pysam output
148
+ data = [row.split("\t")[:10] for row in sam.split("\n")]
149
+ # end = 14 if is_endogenous else 12
150
+ columns = constants.SAMFILE_COLUMNS[:10]
151
+
152
+ # If empty [['']]
153
+ if data[0][0]:
154
+ return pd.DataFrame(data, columns=columns).dropna()
155
+ return pd.DataFrame(columns=columns)
156
+
157
+ def filter_unique_probes(self) -> pd.DataFrame:
158
+ """Filter sam file with probes based on alignment score and uniqueness."""
159
+ # 60 - uniquely mapped read, regardless of number of mismatches / indels
160
+ # 4 – flag for unmapped read
161
+ flags = ["--min-MQ", "60"] if self.is_endogenous else ["--require-flags", "4"]
162
+ self.logger.debug(f"Running samtools with - {' '.join(flags)}")
163
+ filtered_sam = pysam.view(self.fname_sam, *flags) # type: ignore
164
+ df = self.parse_raw_pysam(filtered_sam)
165
+
166
+ # Filter to remove off-target rates
167
+ if self.is_endogenous:
168
+ group_sizes = df.groupby("qname").size()
169
+ df = df[
170
+ df["qname"].isin(
171
+ group_sizes[group_sizes <= self.max_off_targets + 1].index
172
+ )
173
+ ]
174
+ return df
175
+
176
+ # def exclude_gene_of_interest(
177
+ # self, df: pd.DataFrame, ensembl_id: str, fname_full_gene: str, threads: int
178
+ # ) -> pd.DataFrame:
179
+ # """Filter count table to exclude the gene of interest."""
180
+ # # Filter using provided EnsemblID directly
181
+ # if ensembl_id:
182
+ # self.logger.debug(f"Filtering directly using EmsembleID {ensembl_id}")
183
+ # return df[df["gene_id"] != ensembl_id]
184
+
185
+ # # Filter using blast
186
+ # args_blast = [
187
+ # "blastn",
188
+ # "-db",
189
+ # self.fname_genome,
190
+ # "-query",
191
+ # fname_full_gene,
192
+ # "-task",
193
+ # "megablast",
194
+ # "-outfmt",
195
+ # "6",
196
+ # "-num_threads",
197
+ # str(threads),
198
+ # ]
199
+ # self.logger.debug(f"Running blast with - {' '.join(args_blast)}")
200
+ # process = subprocess.run(args_blast, capture_output=True, text=True)
201
+ # data = [row.split("\t") for row in process.stdout.split("\n") if row]
202
+ # df_blast = pd.DataFrame(data, columns=constants.BLAST_COLUMNS).astype(
203
+ # {name: float for name in ["pident", "sstart", "send", "evalue"]}
204
+ # )
205
+
206
+ # # Arbirary identity values to ensure similar targets aren't falsely selected
207
+ # df_blast = df_blast[
208
+ # (df_blast["pident"] >= 98)
209
+ # & (df_blast["evalue"] <= 1e-5)
210
+ # & (df_blast["sseqid"] == df_blast.loc[0, "sseqid"])
211
+ # ]
212
+
213
+ # # Pad blast start/end results in case not everything was found/isoforms/etc.
214
+ # # Set to an arbitrary maximum value of 100 to not include other genes
215
+ # # Account for forward and reverse direction
216
+ # for buffer in range(0, 110, 10):
217
+ # gene_ids = []
218
+ # for _, row in df_blast.iterrows():
219
+ # gene_ids.extend(
220
+ # df[
221
+ # (
222
+ # (df["seqname"] == row["sseqid"])
223
+ # | (df["seqname"] == f"chr{row['sseqid']}")
224
+ # )
225
+ # & ((df["start"] >= row["sstart"]) & (df["end"] <= row["send"]))
226
+ # | ((df["start"] >= row["send"]) & (df["end"] <= row["sstart"]))
227
+ # ]["gene_id"].values
228
+ # )
229
+ # if not gene_ids:
230
+ # self.logger.debug(f"No genes found using a buffer of {buffer}")
231
+ # continue
232
+
233
+ # most_frequent_gene_id = max(set(gene_ids), key=gene_ids.count)
234
+ # self.logger.debug(
235
+ # f"Filtering out most frequent gene {most_frequent_gene_id}"
236
+ # )
237
+ # df = df[df["gene_id"] != most_frequent_gene_id].reset_index(drop=True)
238
+ # break
239
+
240
+ # return df
241
+
242
+ # def join_alignment_with_annotation(
243
+ # self, df_sam: pd.DataFrame, df_norm: pd.DataFrame
244
+ # ) -> pd.DataFrame:
245
+ # """Merge probe alignment with normalized count table based on start/end positions."""
246
+ # dfs = []
247
+
248
+ # for sequence, df_sequence in df_sam.groupby("rname"):
249
+ # df_count_sequence = df_norm[
250
+ # (df_norm["seqname"] == sequence)
251
+ # | (df_norm["seqname"] == sequence.lstrip("chr"))
252
+ # ]
253
+ # gene_start = df_count_sequence["start"].astype(int).values
254
+ # gene_end = df_count_sequence["end"].astype(int).values
255
+ # align_start = df_sequence["pos"].astype(int).values
256
+ # align_length = df_sequence["seq"].apply(len).values
257
+ # align_strand = (
258
+ # (df_sequence["flag"] == constants.SAM_FLAG_REVERSE)
259
+ # .replace({True: -1, False: 1})
260
+ # .values
261
+ # )
262
+ # align_end = align_start + align_strand * align_length
263
+
264
+ # # Create n*m matrix of which alignments(n) match up with which annotations(m)
265
+ # i, j = np.where(
266
+ # (
267
+ # (align_start[:, None] >= gene_start)
268
+ # & (align_start[:, None] <= gene_end)
269
+ # )
270
+ # | (
271
+ # (align_end[:, None] >= gene_start)
272
+ # & (align_end[:, None] <= gene_end)
273
+ # )
274
+ # )
275
+
276
+ # # Join columns to merge alignment information with count values
277
+ # df_sequence = pd.concat(
278
+ # [
279
+ # df_sequence.reset_index(drop=True).iloc[i].reset_index(drop=True),
280
+ # df_count_sequence.reset_index(drop=True)
281
+ # .iloc[j]
282
+ # .reset_index(drop=True),
283
+ # ],
284
+ # axis=1,
285
+ # )
286
+ # dfs.append(df_sequence)
287
+ # dfs = pd.concat(dfs, ignore_index=True)
288
+ # return dfs
289
+
290
+ # def get_most_expressed_genes(self, df: pd.DataFrame, percentage: float) -> Any:
291
+ # """Find the most highly expressed genes based on normalized count."""
292
+ # quantile = df["count"].quantile(percentage / 100)
293
+ # most_expressed = df[df["count"] > quantile].reset_index(drop=True)
294
+ # return most_expressed["gene_id"].values.tolist()
295
+
296
+ # # TODO allow filtering if exogenous?
297
+ # def filter_using_count(self, df_sam: pd.DataFrame) -> pd.DataFrame:
298
+ # """Filter candidates binding off targets above a expression percentage threshold."""
299
+ # df = self.join_alignment_with_annotation(df_sam=df_sam, df_norm=self.norm_table)
300
+ # df.to_csv(self.output()["table"].path, index=False)
301
+ # df_off_targets = self.exclude_gene_of_interest(
302
+ # df,
303
+ # ensembl_id=SequenceConfig().ensembl_id,
304
+ # fname_full_gene=self.input()["blastseq"].path,
305
+ # threads=GeneralConfig().threads,
306
+ # )
307
+ # most_expressed = self.get_most_expressed_genes(
308
+ # df=df_off_targets, percentage=ProbeConfig().max_expression_percentage
309
+ # )
310
+ # df_sam = df[~df["gene_id"].isin(most_expressed)].reset_index(drop=True)
311
+ # return df_sam
312
+
313
+ def run(self):
314
+ # Naming
315
+ self.fname_fasta = self.input()["probes"].path
316
+ self.fname_sam = os.path.splitext(self.fname_fasta)[0] + ".sam"
317
+ self.fname_genome = util.get_genome_name()
318
+
319
+ # Alignment and filtering
320
+ self.align_probes(threads=GeneralConfig().threads)
321
+ df_sam = self.filter_unique_probes()
322
+ df_sam.to_csv(self.output()["table"].path, index=False)
323
+
324
+ # if self.is_blast_required:
325
+ # self.fname_gtf = self.input()["gtf"].path
326
+ # self.fname_count = ProbeConfig().encode_count_table
327
+ # df_sam = self.filter_using_count(df_sam)
328
+
329
+ # Save output
330
+ sequences = list(Bio.SeqIO.parse(self.fname_fasta, "fasta"))
331
+ candidates = [seq for seq in sequences if seq.id in df_sam["qname"].unique()]
332
+ self.logger.debug(df_sam["qname"].values)
333
+ util.log_and_check_candidates(
334
+ self.logger, "AlignProbeCandidates", len(candidates), len(sequences)
335
+ )
336
+ Bio.SeqIO.write(candidates, self.output()["fasta"].path, format="fasta")
eFISHent/analyze.py ADDED
@@ -0,0 +1,246 @@
1
+ """Analyze a probe set and save plots."""
2
+
3
+ from typing import List, Union
4
+ import logging
5
+ import multiprocessing
6
+ import os
7
+
8
+ from mpl_toolkits.axes_grid1 import make_axes_locatable
9
+ import Bio.Seq
10
+ import Bio.SeqIO
11
+ import luigi
12
+ import matplotlib.pyplot as plt
13
+ import numpy as np
14
+ import pysam
15
+
16
+ from . import util
17
+ from .alignment import AlignProbeCandidates
18
+ from .basic_filtering import get_g_quadruplet_count
19
+ from .basic_filtering import get_gc_content
20
+ from .basic_filtering import get_melting_temp
21
+ from .config import GeneralConfig
22
+ from .config import ProbeConfig
23
+ from .config import RunConfig
24
+ from .config import SequenceConfig
25
+ from .indexing import BuildBowtieIndex
26
+ from .kmers import BuildJellyfishIndex
27
+ from .kmers import get_max_kmer_count
28
+ from .prepare_sequence import PrepareSequence
29
+ from .secondary_structure import get_free_energy
30
+
31
+
32
+ class AnalyzeProbeset(luigi.Task):
33
+ """Analysis of a probe set with summary stats."""
34
+
35
+ logger = logging.getLogger("custom-logger")
36
+
37
+ def requires(self):
38
+ tasks = {
39
+ "jellyfish": BuildJellyfishIndex(),
40
+ "bowtie": BuildBowtieIndex(),
41
+ "sequence": PrepareSequence(),
42
+ }
43
+ return tasks
44
+
45
+ def output(self):
46
+ return luigi.LocalTarget(
47
+ os.path.splitext(os.path.basename(RunConfig().analyze_probeset))[0]
48
+ + "_analysis.pdf"
49
+ )
50
+
51
+ @property
52
+ def gene(self):
53
+ """Parse gene sequence file depending on parameters passed."""
54
+ return Bio.SeqIO.read(
55
+ self.input()["sequence"].path, format="fasta"
56
+ ).reverse_complement()
57
+
58
+ @staticmethod
59
+ def histplot(ax: plt.Axes, data: List[int], title: str, min_value: int):
60
+ """Basic narrow count based histogram."""
61
+ ax.set_title(title)
62
+ bins = np.arange(min_value, max(data) + 4) + 0.5
63
+ ax.hist(data, bins=bins)
64
+ ax.set(xticks=bins[1:] - 0.5, ylabel="Count")
65
+
66
+ @staticmethod
67
+ def boxplot(ax: plt.Axes, data: List[Union[int, float]], title: str, ylabel: str):
68
+ """Boxplot with kde-esque lines next to it."""
69
+ ax.set_title(title)
70
+ ax.boxplot(data)
71
+ ax.eventplot(
72
+ data,
73
+ orientation="vertical",
74
+ linewidths=0.2,
75
+ lineoffsets=0.8,
76
+ linelengths=0.1,
77
+ )
78
+ ax.set(xticks=[], ylabel=ylabel)
79
+
80
+ @staticmethod
81
+ def matrix(ax: plt.Axes, data: np.ndarray, title: str, fig): # type: ignore
82
+ """Image with colorbar in same axis."""
83
+ ax.set_title(title)
84
+ image = ax.imshow(data, vmin=0, vmax=1, cmap="coolwarm")
85
+ divider = make_axes_locatable(ax)
86
+ cax = divider.append_axes("right", size="5%", pad=0.05)
87
+ fig.colorbar(image, cax=cax, orientation="vertical")
88
+
89
+ def add_length(self, ax: plt.Axes):
90
+ """Add probe lengths to axis."""
91
+ lengths = list(map(len, self.sequences))
92
+ self.histplot(ax, lengths, "Lengths", min(lengths) - 4)
93
+ self.logger.debug("Added length to axis.")
94
+
95
+ def add_melting_temperature(self, ax: plt.Axes):
96
+ """Add probe melting temperatures to axis."""
97
+ melting_temps = [
98
+ get_melting_temp(
99
+ seq.seq,
100
+ ProbeConfig().na_concentration,
101
+ ProbeConfig().formamide_concentration,
102
+ )
103
+ for seq in self.sequences
104
+ ]
105
+ self.boxplot(ax, melting_temps, "Melting temperatures", "ºC")
106
+ self.logger.debug("Added melting temperature to axis.")
107
+
108
+ def add_gc_content(self, ax: plt.Axes):
109
+ """Add probe GC content to axis."""
110
+ gc_content = [get_gc_content(seq.seq) for seq in self.sequences]
111
+ self.boxplot(ax, gc_content, "GC Content", "%")
112
+ self.logger.debug("Added GC content to axis.")
113
+
114
+ def add_g_quadruplet(self, ax: plt.Axes):
115
+ """Add number of G quadruplets in probes to axis."""
116
+ quads = [get_g_quadruplet_count(seq.seq) for seq in self.sequences]
117
+ self.histplot(ax, quads, "G quadruplet", -1)
118
+ self.logger.debug("Added G quadruplets to axis.")
119
+
120
+ def _get_max_kmer_count(self, seq):
121
+ return get_max_kmer_count(seq, self.jellyfish_path)
122
+ # return get_max_kmer_count(seq, "./tests/data/sacCer3_15.jf")
123
+
124
+ def add_kmers(self, ax: plt.Axes):
125
+ """Add highest number of sub-kmers in probes to axis."""
126
+ self.jellyfish_path = self.input()["jellyfish"].path
127
+ with multiprocessing.Pool(GeneralConfig().threads) as pool:
128
+ kmers = pool.map(self._get_max_kmer_count, self.sequences)
129
+ self.histplot(ax, kmers, "K-mer count (len)", -1)
130
+ self.logger.debug("Added kmers to axis.")
131
+
132
+ def add_free_energy(self, ax: plt.Axes):
133
+ """Add predicted deltaG of probes to axis."""
134
+ with multiprocessing.Pool(GeneralConfig().threads) as pool:
135
+ deltag = pool.map(get_free_energy, self.sequences)
136
+ self.boxplot(ax, deltag, "Free energy", "deltaG")
137
+ self.logger.debug("Added free energy to axis.")
138
+
139
+ def add_off_targets(self, ax: plt.Axes):
140
+ """Add unique number of off targets to axis."""
141
+ task = AlignProbeCandidates()
142
+ task.fname_fasta = RunConfig().analyze_probeset
143
+ task.fname_sam = os.path.splitext(task.fname_fasta)[0] + ".sam"
144
+ task.fname_genome = util.get_genome_name()
145
+ task.fname_gene = util.get_gene_name()
146
+ task.is_endogenous = SequenceConfig().is_endogenous
147
+ task.max_off_targets = 20
148
+
149
+ task.align_probes(threads=GeneralConfig().threads)
150
+ df_sam = task.parse_raw_pysam(pysam.view(task.fname_sam)) # type: ignore
151
+ counts = df_sam.groupby("qname").size().values - 1
152
+ self.histplot(ax, counts, "Off target count", -1)
153
+
154
+ os.remove(task.fname_sam)
155
+ os.remove(task.fname_fasta.rstrip("a") + "q")
156
+ self.logger.debug("Added off targets to axis.")
157
+
158
+ # TODO refactor to merge with optimization.is_binding
159
+ def _get_similarity_score(self, seq1: Bio.Seq.Seq, seq2: Bio.Seq.Seq) -> float:
160
+ alignments = [
161
+ *Bio.pairwise2.align.globalxx(seq1, seq2.complement()),
162
+ *Bio.pairwise2.align.globalxx(seq1, seq2.reverse_complement()),
163
+ ]
164
+ if not alignments:
165
+ return 0.0
166
+ max_score = max(map(lambda x: x.score, alignments)) # type: ignore
167
+ return float(max_score / len(seq1))
168
+
169
+ def add_binding_affinity(self, ax: plt.Axes, fig):
170
+ """Add probe vs probe similarity scores (binding) to axis."""
171
+ if len(self.sequences) >= 25:
172
+ eta = len(self.sequences) ** 2 * 20 / 60_000
173
+ self.logger.warning(
174
+ f"Relatively large number of sequences ({len(self.sequences)}). "
175
+ f"Determining sequence similarity will take ~{eta:.2f}mins."
176
+ )
177
+ matrix = np.array(
178
+ [
179
+ [self._get_similarity_score(x.seq, y.seq) for x in self.sequences]
180
+ for y in self.sequences
181
+ ]
182
+ ) # type: ignore
183
+ self.matrix(ax, matrix, "Binding affinity", fig)
184
+ self.logger.debug("Added binding affinity to axis.")
185
+
186
+ def add_probe_coverage(self, ax: plt.Axes):
187
+ """Add coverage along gene to axis."""
188
+ coverage = np.zeros(len(self.gene))
189
+
190
+ for probe in self.sequences:
191
+ idx = self.gene.seq.find(str(probe.seq))
192
+ coverage[idx : idx + len(probe)] = 1
193
+
194
+ ax.set_title("Gene coverage")
195
+ ax.bar(range(len(self.gene)), coverage)
196
+ ax.set(ylabel="Assigned", xlabel="Position")
197
+ self.logger.debug("Added probe coverage to axis.")
198
+
199
+ def build_figure(self):
200
+ """Layout to build the figure."""
201
+ fig = plt.figure(figsize=(15, 18))
202
+ plt.suptitle(
203
+ f"Analysis of probe set - {util.get_gene_name(False)}", y=1, fontsize=20
204
+ )
205
+ shape = (4, 3)
206
+
207
+ # Row 1
208
+ ax = plt.subplot2grid(shape, (0, 0))
209
+ self.add_length(ax)
210
+ ax = plt.subplot2grid(shape, (0, 1))
211
+ self.add_melting_temperature(ax)
212
+ ax = plt.subplot2grid(shape, (0, 2))
213
+ self.add_gc_content(ax)
214
+
215
+ # Row 2
216
+ ax = plt.subplot2grid(shape, (1, 0))
217
+ self.add_g_quadruplet(ax)
218
+ ax = plt.subplot2grid(shape, (1, 1))
219
+ self.add_kmers(ax)
220
+ ax = plt.subplot2grid(shape, (1, 2))
221
+ self.add_free_energy(ax)
222
+
223
+ # Row 3
224
+ ax = plt.subplot2grid(shape, (2, 0))
225
+ self.add_off_targets(ax)
226
+ ax = plt.subplot2grid(shape, (2, 1))
227
+ self.add_binding_affinity(ax, fig)
228
+
229
+ # TODO add a way to display off target expression percentages?
230
+ # if ProbeConfig().encode_count_table:
231
+ # ax = plt.subplot2grid(shape, (2, 2))
232
+ # self.add_expression_percentages(ax)
233
+
234
+ # Row 4
235
+ ax = plt.subplot2grid(shape, (3, 0), colspan=3)
236
+ self.add_probe_coverage(ax)
237
+
238
+ plt.tight_layout()
239
+ plt.savefig(self.output().path)
240
+
241
+ def run(self):
242
+ self.sequences = list(
243
+ Bio.SeqIO.parse(RunConfig().analyze_probeset, format="fasta")
244
+ )
245
+ self.build_figure()
246
+ os.remove(self.input()["sequence"].path)