methplotlib 0.20.1__tar.gz → 0.21.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. methplotlib-0.21.2/LICENSE +21 -0
  2. methplotlib-0.21.2/PKG-INFO +93 -0
  3. {methplotlib-0.20.1 → methplotlib-0.21.2}/README.md +8 -3
  4. {methplotlib-0.20.1 → methplotlib-0.21.2}/README.rst +9 -11
  5. methplotlib-0.21.2/methplotlib/import_methylation.py +482 -0
  6. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/methplotlib.py +88 -56
  7. methplotlib-0.21.2/methplotlib/plots.py +456 -0
  8. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/utils.py +20 -4
  9. methplotlib-0.21.2/methplotlib/version.py +1 -0
  10. methplotlib-0.21.2/methplotlib.egg-info/PKG-INFO +93 -0
  11. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/SOURCES.txt +1 -0
  12. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/entry_points.txt +0 -1
  13. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/requires.txt +1 -1
  14. methplotlib-0.21.2/setup.py +57 -0
  15. methplotlib-0.20.1/PKG-INFO +0 -88
  16. methplotlib-0.20.1/methplotlib/import_methylation.py +0 -305
  17. methplotlib-0.20.1/methplotlib/plots.py +0 -384
  18. methplotlib-0.20.1/methplotlib/version.py +0 -1
  19. methplotlib-0.20.1/methplotlib.egg-info/PKG-INFO +0 -88
  20. methplotlib-0.20.1/setup.py +0 -51
  21. {methplotlib-0.20.1 → methplotlib-0.21.2}/MANIFEST.in +0 -0
  22. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/__init__.py +0 -0
  23. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/annotation.py +0 -0
  24. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/differential/__init__.py +0 -0
  25. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/differential/differential.py +0 -0
  26. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/helpers.py +0 -0
  27. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/qc.py +0 -0
  28. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/dependency_links.txt +0 -0
  29. {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/top_level.txt +0 -0
  30. {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/__init__.py +0 -0
  31. {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/allele_specific_modification +0 -0
  32. {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/annotate_calls_by_phase.py +0 -0
  33. {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/calculate_methylation_frequency.py +0 -0
  34. {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/differential_modification +0 -0
  35. {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/genome-wide-QC.py +0 -0
  36. {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/sorting_and_multiple_testing_correction.py +0 -0
  37. {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/split_calls_by_longshot_phase.py +0 -0
  38. {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/split_calls_by_phase.py +0 -0
  39. {methplotlib-0.20.1 → methplotlib-0.21.2}/setup.cfg +0 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2018 Wouter De Coster
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.1
2
+ Name: methplotlib
3
+ Version: 0.21.2
4
+ Summary: Plot methylation data obtained from nanopolish
5
+ Home-page: https://github.com/wdecoster/methplotlib
6
+ Author: Wouter De Coster
7
+ Author-email: decosterwouter@gmail.com
8
+ License: MIT
9
+ Keywords: nanopore
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.3
16
+ Classifier: Programming Language :: Python :: 3.4
17
+ Classifier: Programming Language :: Python :: 3.5
18
+ Requires-Python: >=3
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+
22
+ [![Anaconda-Server Badge](https://anaconda.org/bioconda/methplotlib/badges/version.svg)](https://anaconda.org/bioconda/methplotlib)
23
+ [![Anaconda-Server Badge](https://anaconda.org/bioconda/methplotlib/badges/license.svg)](https://anaconda.org/bioconda/methplotlib)
24
+
25
+ # METHPLOTLIB
26
+
27
+ This script generates a browser view on a window using data from
28
+ i) [nanopolish](https://github.com/jts/nanopolish), either as methylation calls or methylation frequencies (as processed by calculate_methylation_frequency.py). The methylation calls can additionally be phased using scripts/annotate_calls_by_phase.and scripts/split_calls_by_phase.py
29
+ ii) [nanocompore](https://github.com/tleonardi/nanocompore)
30
+ iii) in ont-cram format with MM/ML tags according to the SAM specifications
31
+ iv) in bedgraph format
32
+
33
+ ## INSTALLATION
34
+
35
+ Creating a new conda environment:
36
+ `conda create -n methplotlib methplotlib`
37
+
38
+ Or using pip:
39
+ `pip install methplotlib`
40
+
41
+
42
+
43
+ ## USAGE
44
+ ```
45
+ methplotlib [-h] [-v] -m METHYLATION [METHYLATION ...] -n NAMES
46
+ [NAMES ...] -w WINDOW [-g GTF] [-b BED] [-f FASTA]
47
+ [--simplify] [--split] [--static STATIC] [--smooth SMOOTH]
48
+ [--dotsize DOTSIZE] [--example] [-o OUTFILE] [-q QCFILE]
49
+
50
+ plotting nanopolish methylation calls or frequency
51
+
52
+ optional arguments:
53
+ -h, --help show this help message and exit
54
+ -v, --version Print version and exit.
55
+ -m, --methylation METHYLATION [METHYLATION ...]
56
+ data in nanopolish, nanocompore, ont-cram or bedgraph
57
+ format
58
+ -n, --names NAMES [NAMES ...]
59
+ names of datasets in --methylation
60
+ -w, --window WINDOW window (region) to which the visualisation has to be restricted
61
+ -g, --gtf GTF add annotation based on a gtf file
62
+ -b, --bed BED add annotation based on a bed file
63
+ -f, --fasta FASTA required when --window is an entire chromosome, contig or transcript
64
+ --simplify simplify annotation track to show genes rather than transcripts
65
+ --split split, rather than overlay the methylation tracks
66
+ --static Make a static image of the browser window (filename)
67
+ --binary Make the nanopolish plot ignorning log likelihood nuances
68
+ --smooth Rolling window size for averaging frequency values (int)
69
+ --dotsize Control the size of dots in the per read plots (int)
70
+ --example Show example command and exit.
71
+ -o, --outfile OUTFILE File to write results to. Default:
72
+ methylation_browser_{chr}_{start}_{end}.html. Use
73
+ {region} as a shorthand for {chr}_{start}_{end} in the
74
+ filename. Missing paths will be created.
75
+ -q, --qcfile QCFILE File to write the qc report to. Default: The path in
76
+ outfile prefixed with qc_, default is qc_report_methyl
77
+ ation_browser_{chr}_{start}_{end}.html. Use {region}
78
+ as a shorthand for {chr}_{start}_{end} in the
79
+ filename. Missing paths will be created.
80
+
81
+ ```
82
+
83
+ ## Snakemake workflow
84
+ For streamlining nanopolish a Snakefile is included (using snakemake). The workflow uses a config file, of which an example is in this repository.
85
+
86
+ ## Example data
87
+ The `examples` folder contains calls and frequencies for the human ACTB gene from PromethION sequencing of NA19240. An example command is available.
88
+
89
+ ## Companion scripts
90
+ The `scripts` folder contains scripts for phasing modification calls in haplotypes based on [WhatsHap](https://whatshap.readthedocs.io/en/latest/) phasing, allele specific modification testing for phased data and differential modification testing across subjects.
91
+
92
+ ## TO DO - CONTRIBUTIONS WELCOME
93
+ - Outlier detection (in windows) across samples
@@ -1,5 +1,3 @@
1
- [![Build Status](https://travis-ci.com/wdecoster/methplotlib.svg?branch=master)](https://travis-ci.com/wdecoster/methplotlib)
2
- [![Anaconda-Server Badge](https://anaconda.org/bioconda/methplotlib/badges/installer/conda.svg)](https://conda.anaconda.org/bioconda)
3
1
  [![Anaconda-Server Badge](https://anaconda.org/bioconda/methplotlib/badges/version.svg)](https://anaconda.org/bioconda/methplotlib)
4
2
  [![Anaconda-Server Badge](https://anaconda.org/bioconda/methplotlib/badges/license.svg)](https://anaconda.org/bioconda/methplotlib)
5
3
 
@@ -8,12 +6,19 @@
8
6
  This script generates a browser view on a window using data from
9
7
  i) [nanopolish](https://github.com/jts/nanopolish), either as methylation calls or methylation frequencies (as processed by calculate_methylation_frequency.py). The methylation calls can additionally be phased using scripts/annotate_calls_by_phase.and scripts/split_calls_by_phase.py
10
8
  ii) [nanocompore](https://github.com/tleonardi/nanocompore)
11
- iii) in ont-cram format with MM/MP tags according to the SAM specifications and converted using e.g. [this script](https://github.com/kpalin/gcf52ref/blob/f5_to_usam/scripts/extract_methylation_fast5_to_sam.py)
9
+ iii) in ont-cram format with MM/ML tags according to the SAM specifications
12
10
  iv) in bedgraph format
13
11
 
14
12
  ## INSTALLATION
13
+
14
+ Creating a new conda environment:
15
+ `conda create -n methplotlib methplotlib`
16
+
17
+ Or using pip:
15
18
  `pip install methplotlib`
16
19
 
20
+
21
+
17
22
  ## USAGE
18
23
  ```
19
24
  methplotlib [-h] [-v] -m METHYLATION [METHYLATION ...] -n NAMES
@@ -1,5 +1,4 @@
1
- |Build Status| |Anaconda-Server Badge| |Anaconda-Server Badge|
2
- |Anaconda-Server Badge|
1
+ |Anaconda-Server Badge| |image1|
3
2
 
4
3
  METHPLOTLIB
5
4
  ===========
@@ -11,15 +10,18 @@ METHPLOTLIB
11
10
  additionally be phased using scripts/annotate_calls_by_phase.and
12
11
  scripts/split_calls_by_phase.py
13
12
  | ii) `nanocompore <https://github.com/tleonardi/nanocompore>`__
14
- | iii) in ont-cram format with MM/MP tags according to the SAM
15
- specifications and converted using e.g. `this
16
- script <https://github.com/kpalin/gcf52ref/blob/f5_to_usam/scripts/extract_methylation_fast5_to_sam.py>`__
13
+ | iii) in ont-cram format with MM/ML tags according to the SAM
14
+ specifications
17
15
  | iv) in bedgraph format
18
16
 
19
17
  INSTALLATION
20
18
  ------------
21
19
 
22
- ``pip install methplotlib``
20
+ | Creating a new conda environment:
21
+ | ``conda create -n methplotlib methplotlib``
22
+
23
+ | Or using pip:
24
+ | ``pip install methplotlib``
23
25
 
24
26
  USAGE
25
27
  -----
@@ -90,11 +92,7 @@ TO DO - CONTRIBUTIONS WELCOME
90
92
 
91
93
  - Outlier detection (in windows) across samples
92
94
 
93
- .. |Build Status| image:: https://travis-ci.com/wdecoster/methplotlib.svg?branch=master
94
- :target: https://travis-ci.com/wdecoster/methplotlib
95
- .. |Anaconda-Server Badge| image:: https://anaconda.org/bioconda/methplotlib/badges/installer/conda.svg
96
- :target: https://conda.anaconda.org/bioconda
97
95
  .. |Anaconda-Server Badge| image:: https://anaconda.org/bioconda/methplotlib/badges/version.svg
98
96
  :target: https://anaconda.org/bioconda/methplotlib
99
- .. |Anaconda-Server Badge| image:: https://anaconda.org/bioconda/methplotlib/badges/license.svg
97
+ .. |image1| image:: https://anaconda.org/bioconda/methplotlib/badges/license.svg
100
98
  :target: https://anaconda.org/bioconda/methplotlib
@@ -0,0 +1,482 @@
1
+ import pandas as pd
2
+ import pyranges as pr
3
+ import numpy as np
4
+ import sys
5
+ import logging
6
+ from methplotlib.utils import file_sniffer, flatten
7
+ from itertools import repeat
8
+
9
+
10
+ class Modification(object):
11
+ def __init__(self, table, data_type, name, called_sites, start_end_table=None):
12
+ self.table = table
13
+ self.data_type = data_type
14
+ self.name = name
15
+ self.called_sites = called_sites
16
+ self.start_end_table = start_end_table
17
+
18
+
19
+ def get_data(methylation_files, names, window, smoothen=5):
20
+ """
21
+ Import methylation data from all files in the list methylation_files
22
+
23
+ Data can in various formats
24
+ - nanopolish raw, phased, frequency
25
+ - cram
26
+ - bam
27
+ - nanocompore
28
+ - bedgraph
29
+ - bedmethyl in the extended form (as obtained from modbam2bed --extended)
30
+
31
+ data is extracted within the window args.window
32
+ Frequencies are smoothened using a sliding window
33
+ """
34
+ return flatten([read_mods(f, n, window, smoothen) for f, n in zip(methylation_files, names)])
35
+
36
+
37
+ def read_mods(filename, name, window, smoothen=5):
38
+ """
39
+ converts a file from nanopolish to a pandas dataframe
40
+ input can be from calculate_methylation_frequency
41
+ which will return a dataframe with 'chromosome', 'pos', 'methylated_frequency'
42
+ smoothening the result by a rolling average
43
+
44
+ input can also be raw data per read, optionally phased
45
+ which will return a dataframe with 'read', 'chromosome', 'pos', 'log_lik_ratio', 'strand'
46
+ """
47
+ file_type = file_sniffer(filename)
48
+ logging.info(f"File {filename} is of type {file_type}")
49
+ try:
50
+ if file_type.startswith("nanopolish"):
51
+ return parse_nanopolish(filename, file_type, name, window, smoothen=smoothen)
52
+ elif file_type == "nanocompore":
53
+ return [parse_nanocompore(filename, name, window)]
54
+ elif file_type in ["cram", "bam"]:
55
+ return parse_cram(filename, file_type, name, window)
56
+ elif file_type == "bedgraph":
57
+ return [parse_bedgraph(filename, name, window)]
58
+ elif file_type == "bedmethyl_extended":
59
+ return [parse_bedmethyl(filename, name, window, smoothen=smoothen, flavor="modbam2bed")]
60
+ elif file_type == "bedmethyl":
61
+ return [parse_bedmethyl(filename, name, window, smoothen=smoothen, flavor="modkit")]
62
+ except Exception as e:
63
+ logging.error(f"Error processing {filename}.")
64
+ logging.error(e, exc_info=True)
65
+ sys.stderr.write(f"\n\n\nError processing {filename}!\n")
66
+ sys.stderr.write("\n\n\nDetailed error:\n")
67
+ raise
68
+
69
+
70
+ def parse_nanopolish(filename, file_type, name, window, smoothen=5):
71
+ if window:
72
+ from pathlib import Path
73
+
74
+ if Path(filename + ".tbi").is_file():
75
+ import subprocess
76
+ import gzip
77
+
78
+ logging.info(f"Reading {filename} using a tabix stream.")
79
+ region = f"{window.chromosome}:{window.begin}-{window.end}"
80
+ try:
81
+ tabix_stream = subprocess.Popen(
82
+ ["tabix", filename, region],
83
+ stdout=subprocess.PIPE,
84
+ stderr=subprocess.PIPE,
85
+ )
86
+ except FileNotFoundError as e:
87
+ logging.error("Error when opening a tabix stream.")
88
+ logging.error(e, exc_info=True)
89
+ sys.stderr.write("\n\nERROR when opening a tabix stream.\n")
90
+ sys.stderr.write("Is tabix installed and on the PATH?.")
91
+ raise
92
+ header = gzip.open(filename, "rt").readline().rstrip().split("\t")
93
+ table = pd.read_csv(tabix_stream.stdout, sep="\t", header=None, names=header)
94
+ else:
95
+ logging.info(f"Reading {filename} slowly by splitting the file in chunks.")
96
+ sys.stderr.write(f"\nReading {filename} would be faster with bgzip and tabix.\n")
97
+ if file_type in ["nanopolish_call", "nanopolish_phased"]:
98
+ sys.stderr.write("Please index with 'tabix -S1 -s1 -b3 -e4'.\n")
99
+ else:
100
+ sys.stderr.write("Please index with 'tabix -S1 -s1 -b2 -e3'.\n")
101
+ iter_csv = pd.read_csv(filename, sep="\t", iterator=True, chunksize=1e6)
102
+ table = pd.concat(
103
+ [chunk[chunk["chromosome"] == window.chromosome] for chunk in iter_csv]
104
+ )
105
+ else:
106
+ table = pd.read_csv(filename, sep="\t")
107
+ gr = pr.PyRanges(
108
+ table.rename(
109
+ columns={
110
+ "start": "Start",
111
+ "chromosome": "Chromosome",
112
+ "end": "End",
113
+ "Strand": "strand",
114
+ }
115
+ )
116
+ )
117
+ logging.info("Read the file in a dataframe.")
118
+
119
+ if window:
120
+ gr = gr[str(window.chromosome), window.begin : window.end]
121
+ try:
122
+ gr.pos = np.floor(gr.drop().df[["Start", "End"]].mean(axis=1))
123
+ except KeyError:
124
+ sys.stderr.write(f"\n\n\nProblem parsing nanopolish file {filename}!\n")
125
+ sys.stderr.write(
126
+ "Could it be that there are no calls in your selected window "
127
+ f"{window.chromosome}:{window.begin}-{window.end}?\n"
128
+ )
129
+ sys.stderr.write("\n\n\nDetailed error:\n")
130
+ raise
131
+
132
+ table = gr.df
133
+
134
+ if file_type in ["nanopolish_call", "nanopolish_phased"]:
135
+ table = table.drop(
136
+ columns=[
137
+ "Start",
138
+ "End",
139
+ "log_lik_methylated",
140
+ "log_lik_unmethylated",
141
+ "num_calling_strands",
142
+ "num_motifs",
143
+ "sequence",
144
+ ]
145
+ )
146
+ if "motif" in table:
147
+ return [
148
+ Modification(
149
+ table=sub_df,
150
+ data_type=file_type,
151
+ name=f"{name}_{mod}",
152
+ called_sites=len(sub_df),
153
+ )
154
+ for mod, sub_df in table.groupby("motif")
155
+ ]
156
+ else:
157
+ return [
158
+ Modification(
159
+ table=table.sort_values(["read_name", "pos"]),
160
+ data_type=file_type,
161
+ name=name,
162
+ called_sites=len(table),
163
+ )
164
+ ]
165
+ if file_type == "nanopolish_freq":
166
+ called_sites = table.called_sites
167
+ chromosome = table.Chromosome.values[0]
168
+
169
+ table = table.drop(
170
+ columns=[
171
+ "Chromosome",
172
+ "Start",
173
+ "End",
174
+ "num_motifs_in_group",
175
+ "called_sites",
176
+ "called_sites_methylated",
177
+ "group_sequence",
178
+ ]
179
+ )
180
+ return [
181
+ Modification(
182
+ table=table.sort_values("pos")
183
+ .groupby("pos")
184
+ .mean()
185
+ .rolling(window=smoothen, center=True)
186
+ .mean()
187
+ .assign(Chromosome=chromosome),
188
+ data_type=file_type,
189
+ name=name,
190
+ called_sites=called_sites.sum(),
191
+ )
192
+ ]
193
+
194
+
195
+ def parse_nanocompore(filename, name, window):
196
+ def nanocompore_columns_of_interest(column):
197
+ if column in ["pos", "ref_id"]:
198
+ return True
199
+ elif column.endswith("pvalue_context_2") or column.endswith("pvalue"):
200
+ return True
201
+ else:
202
+ return False
203
+
204
+ table = pd.read_csv(filename, sep="\t", usecols=nanocompore_columns_of_interest)
205
+ if window:
206
+ table = table[table["ref_id"] == window.chromosome]
207
+ return Modification(
208
+ table=table.sort_values("pos")
209
+ .append({"pos": window.end}, ignore_index=True)
210
+ .drop(columns="ref_id")
211
+ .fillna(1.0),
212
+ data_type="nanocompore",
213
+ name=name,
214
+ called_sites=len(table),
215
+ )
216
+
217
+
218
+ def parse_bedgraph(filename, name, window):
219
+ if window:
220
+ from pathlib import Path
221
+
222
+ if Path(filename + ".tbi").is_file():
223
+ import subprocess
224
+
225
+ logging.info(f"Reading {filename} using a tabix stream.")
226
+ region = f"{window.chromosome}:{window.begin}-{window.end}"
227
+ try:
228
+ tabix_stream = subprocess.Popen(
229
+ ["tabix", filename, region],
230
+ stdout=subprocess.PIPE,
231
+ stderr=subprocess.PIPE,
232
+ )
233
+ except FileNotFoundError as e:
234
+ logging.error("Error when opening a tabix stream.")
235
+ logging.error(e, exc_info=True)
236
+ sys.stderr.write("\n\nERROR when opening a tabix stream.\n")
237
+ sys.stderr.write("Is tabix installed and on the PATH?.")
238
+ raise
239
+ table = pd.read_csv(
240
+ tabix_stream.stdout,
241
+ sep="\t",
242
+ header=None,
243
+ names=["Chromosome", "Start", "End", "Value"],
244
+ )
245
+ else:
246
+ logging.info(f"Reading {filename} slowly by splitting the file in chunks.")
247
+ sys.stderr.write(
248
+ f"\nReading {filename} would be faster with bgzip and 'tabix -p bed'.\n"
249
+ )
250
+ iter_csv = pd.read_csv(
251
+ filename,
252
+ sep="\t",
253
+ iterator=True,
254
+ chunksize=1e6,
255
+ header=None,
256
+ names=["Chromosome", "Start", "End", "Value"],
257
+ )
258
+ table = pd.concat(
259
+ [chunk[chunk["Chromosome"] == window.chromosome] for chunk in iter_csv]
260
+ )
261
+ else:
262
+ table = pd.read_csv(
263
+ filename,
264
+ sep="\t",
265
+ header=None,
266
+ names=["Chromosome", "Start", "End", "Value"],
267
+ )
268
+ gr = pr.PyRanges(table)
269
+ logging.info("Read the file in a dataframe.")
270
+ if window:
271
+ gr = gr[window.chromosome, window.begin : window.end]
272
+ if len(gr.df) == 0:
273
+ sys.exit(f"No records for {filename} in {window.string}!\n")
274
+ return Modification(
275
+ table=gr.df.sort_values("Start"),
276
+ data_type="bedgraph",
277
+ name=name,
278
+ called_sites=len(table),
279
+ )
280
+
281
+
282
+ def parse_bedmethyl(filename, name, window, smoothen=5, flavor="modkit"):
283
+ if not flavor in ["modkit", "modbam2bed"]:
284
+ sys.exit(f"ERROR: flavor {flavor} not supported for bedmethyl.")
285
+ if flavor == "modkit":
286
+ colnames = {
287
+ 0: "Chromosome",
288
+ 1: "Start",
289
+ 2: "End",
290
+ # if multiple types of modification should be supported additionally column 3 has to be read
291
+ # 3: "Modification",
292
+ 10: "Frequency",
293
+ }
294
+ else:
295
+ colnames = {
296
+ 0: "Chromosome",
297
+ 1: "Start",
298
+ 2: "End",
299
+ # if multiple types of modification should be supported additionally column 3 has to be read
300
+ # 3: "Modification",
301
+ 11: "canonical",
302
+ 12: "modified",
303
+ }
304
+ usecols = colnames.keys()
305
+ if window:
306
+ from pathlib import Path
307
+
308
+ if Path(filename + ".tbi").is_file():
309
+ import subprocess
310
+
311
+ logging.info(f"Reading {filename} using a tabix stream.")
312
+ region = f"{window.chromosome}:{window.begin}-{window.end}"
313
+ try:
314
+ tabix_stream = subprocess.Popen(
315
+ ["tabix", filename, region],
316
+ stdout=subprocess.PIPE,
317
+ stderr=subprocess.PIPE,
318
+ )
319
+ except FileNotFoundError as e:
320
+ logging.error("Error when opening a tabix stream.")
321
+ logging.error(e, exc_info=True)
322
+ sys.stderr.write("\n\nERROR when opening a tabix stream.\n")
323
+ sys.stderr.write("Is tabix installed and on the PATH?.")
324
+ raise
325
+ table = pd.read_csv(
326
+ tabix_stream.stdout,
327
+ sep="\t",
328
+ header=None,
329
+ usecols=usecols,
330
+ ).rename(columns=colnames)
331
+ else:
332
+ logging.info(f"Reading {filename} slowly by splitting the file in chunks.")
333
+ sys.stderr.write(
334
+ f"\nReading {filename} would be faster with bgzip and 'tabix -p bed'.\n"
335
+ )
336
+ iter_csv = pd.read_csv(
337
+ filename,
338
+ sep="\t",
339
+ iterator=True,
340
+ chunksize=1e6,
341
+ header=None,
342
+ usecols=usecols,
343
+ )
344
+ table = pd.concat([chunk[chunk[0] == window.chromosome] for chunk in iter_csv]).rename(
345
+ columns=colnames
346
+ )
347
+ else:
348
+ table = pd.read_csv(
349
+ filename,
350
+ sep="\t",
351
+ header=None,
352
+ usecols=usecols,
353
+ ).rename(columns=colnames)
354
+ gr = pr.PyRanges(table)
355
+ logging.info("Read the file in a dataframe.")
356
+ if window:
357
+ gr = gr[window.chromosome, window.begin : window.end]
358
+ if len(gr.df) == 0:
359
+ sys.exit(f"No records for {filename} in {window.string}!\n")
360
+ table = gr.df.sort_values("Start")
361
+ if flavor == "modkit":
362
+ table["modified_frequency"] = table["Frequency"] / 100
363
+ table.drop(columns=["Frequency"], inplace=True)
364
+ else:
365
+ table["modified_frequency"] = table["modified"] / (table["canonical"] + table["modified"])
366
+ table.drop(
367
+ columns=["Chromosome", "End", "canonical", "modified"], inplace=True, errors="ignore"
368
+ )
369
+ return Modification(
370
+ table=table.rolling(window=smoothen, center=True).mean(),
371
+ data_type="bedmethyl_extended",
372
+ name=name,
373
+ called_sites=len(table),
374
+ )
375
+
376
+
377
+ def parse_cram(filename, filetype, name, window):
378
+ import pysam
379
+
380
+ mode = "rc" if filetype == "cram" else "rb"
381
+ cram = pysam.AlignmentFile(filename, mode)
382
+ data = []
383
+ start_stops = []
384
+ for read in cram.fetch(reference=str(window.chromosome), start=window.begin, end=window.end):
385
+ if not read.is_supplementary and not read.is_secondary:
386
+ start_stops.append((read.query_name, read.reference_start, read.reference_end))
387
+ mod_positions = get_modified_reference_positions(read)
388
+ if mod_positions:
389
+ data.extend(mod_positions)
390
+ df = (
391
+ pd.DataFrame(data, columns=["read_name", "strand", "pos", "quality", "mod"])
392
+ .astype(dtype={"mod": "category", "quality": "float"})
393
+ .sort_values(["read_name", "pos"])
394
+ )
395
+
396
+ return [
397
+ Modification(
398
+ table=sub_df,
399
+ data_type="ont-cram",
400
+ name=f"{name}_{mod}",
401
+ called_sites=len(sub_df),
402
+ start_end_table=pd.DataFrame(
403
+ start_stops, columns=["read_name", "posmin", "posmax"]
404
+ ).set_index("read_name"),
405
+ )
406
+ for mod, sub_df in df.groupby("mod")
407
+ ]
408
+
409
+
410
+ def get_modified_reference_positions(read):
411
+ mod_positions = []
412
+ if read.has_tag("Mm") or read.has_tag("MM"):
413
+ modtag = "Mm" if read.has_tag("Mm") else "MM"
414
+ qualtag = "Ml" if read.has_tag("Ml") else "ML"
415
+ offset = 0
416
+ for context in read.get_tag(modtag).split(";"):
417
+ if context:
418
+ basemod = context.split(",", 1)[0]
419
+ if "-" in basemod:
420
+ sys.exit(
421
+ "ERROR: modifications on negative strand currently unsupported.\n"
422
+ "Please contact me if this would be of interest for you."
423
+ )
424
+ base, mod = basemod.split("+")
425
+ # code below does not work with for ambiguous N and will search for a literal N
426
+ if base == "N":
427
+ sys.exit(
428
+ "ERROR: modifications of N nucleotides currently unsupported.\n"
429
+ "Please contact me if this would be of interest for you."
430
+ )
431
+ # The positions are encoded by specifying the number of non-modified occurences
432
+ # of that specific bases to skip in the read sequence
433
+ deltas = [int(i) for i in context.split(",")[1:]]
434
+ if not deltas:
435
+ continue
436
+ # Make an array with all positions for which a modification is reported
437
+ # In coordinates relative to occurences of the nucleotide in the read
438
+ locations = np.cumsum(deltas) + np.arange(len(deltas))
439
+ # Nake an array with all read positions for which that nucleotide exists in the read
440
+ # Oddly, get_forward_sequence() returns the sequence as it came from the sequencer
441
+ # This is the same direction as the Mm/Ml tags are specified in
442
+ base_index = np.array(
443
+ [i for i, letter in enumerate(read.get_forward_sequence()) if letter == base]
444
+ )
445
+ # Based on both arrays, find those read coordinates that have a modification
446
+ modified_bases = base_index[locations]
447
+ # Convert the array of read coordinates to reference coordinates
448
+ # full_length=True will make sure the list is of the same length as the read
449
+ # by adding 'None' for softclipped nucleotides
450
+ refpos = np.array(read.get_reference_positions(full_length=True))
451
+ # read.get_reference_positions() returns incrementing coordinates
452
+ # regardless of strand, so reversing the order for reverse_stranded reads
453
+ if read.is_reverse:
454
+ refpos = refpos[::-1]
455
+ # The likelihoods are in an array of length of all Mm/MM deltas,
456
+ # and are not separated by context/modified nucleotide type
457
+ if read.has_tag(qualtag):
458
+ likelihoods = read.get_tag(qualtag).tolist()[offset : offset + len(deltas)]
459
+ else:
460
+ likelihoods = [255] * len(deltas)
461
+ offset += len(deltas)
462
+ mod_positions.extend(
463
+ zip(
464
+ repeat(read.query_name), # read_name
465
+ repeat("-" if read.is_reverse else "+"), # strand
466
+ refpos[modified_bases], # pos
467
+ likelihoods, # quality
468
+ repeat(basemod), # mod
469
+ )
470
+ )
471
+ return mod_positions
472
+ else:
473
+ return None
474
+
475
+
476
+ def errs_tab(n):
477
+ """Generate list of error rates for qualities less than equal than n."""
478
+ return [10 ** (q / -10) for q in range(n + 1)]
479
+
480
+
481
+ def phred_to_probability(quals, tab=errs_tab(128)):
482
+ return [tab[ord(q) - 33] for q in quals]