methplotlib 0.20.1__tar.gz → 0.21.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- methplotlib-0.21.2/LICENSE +21 -0
- methplotlib-0.21.2/PKG-INFO +93 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/README.md +8 -3
- {methplotlib-0.20.1 → methplotlib-0.21.2}/README.rst +9 -11
- methplotlib-0.21.2/methplotlib/import_methylation.py +482 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/methplotlib.py +88 -56
- methplotlib-0.21.2/methplotlib/plots.py +456 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/utils.py +20 -4
- methplotlib-0.21.2/methplotlib/version.py +1 -0
- methplotlib-0.21.2/methplotlib.egg-info/PKG-INFO +93 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/SOURCES.txt +1 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/entry_points.txt +0 -1
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/requires.txt +1 -1
- methplotlib-0.21.2/setup.py +57 -0
- methplotlib-0.20.1/PKG-INFO +0 -88
- methplotlib-0.20.1/methplotlib/import_methylation.py +0 -305
- methplotlib-0.20.1/methplotlib/plots.py +0 -384
- methplotlib-0.20.1/methplotlib/version.py +0 -1
- methplotlib-0.20.1/methplotlib.egg-info/PKG-INFO +0 -88
- methplotlib-0.20.1/setup.py +0 -51
- {methplotlib-0.20.1 → methplotlib-0.21.2}/MANIFEST.in +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/__init__.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/annotation.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/differential/__init__.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/differential/differential.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/helpers.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib/qc.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/dependency_links.txt +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/methplotlib.egg-info/top_level.txt +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/__init__.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/allele_specific_modification +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/annotate_calls_by_phase.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/calculate_methylation_frequency.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/differential_modification +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/genome-wide-QC.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/sorting_and_multiple_testing_correction.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/split_calls_by_longshot_phase.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/scripts/split_calls_by_phase.py +0 -0
- {methplotlib-0.20.1 → methplotlib-0.21.2}/setup.cfg +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2018 Wouter De Coster
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: methplotlib
|
|
3
|
+
Version: 0.21.2
|
|
4
|
+
Summary: Plot methylation data obtained from nanopolish
|
|
5
|
+
Home-page: https://github.com/wdecoster/methplotlib
|
|
6
|
+
Author: Wouter De Coster
|
|
7
|
+
Author-email: decosterwouter@gmail.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Keywords: nanopore
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.4
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.5
|
|
18
|
+
Requires-Python: >=3
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
|
|
22
|
+
[](https://anaconda.org/bioconda/methplotlib)
|
|
23
|
+
[](https://anaconda.org/bioconda/methplotlib)
|
|
24
|
+
|
|
25
|
+
# METHPLOTLIB
|
|
26
|
+
|
|
27
|
+
This script generates a browser view on a window using data from
|
|
28
|
+
i) [nanopolish](https://github.com/jts/nanopolish), either as methylation calls or methylation frequencies (as processed by calculate_methylation_frequency.py). The methylation calls can additionally be phased using scripts/annotate_calls_by_phase.and scripts/split_calls_by_phase.py
|
|
29
|
+
ii) [nanocompore](https://github.com/tleonardi/nanocompore)
|
|
30
|
+
iii) in ont-cram format with MM/ML tags according to the SAM specifications
|
|
31
|
+
iv) in bedgraph format
|
|
32
|
+
|
|
33
|
+
## INSTALLATION
|
|
34
|
+
|
|
35
|
+
Creating a new conda environment:
|
|
36
|
+
`conda create -n methplotlib methplotlib`
|
|
37
|
+
|
|
38
|
+
Or using pip:
|
|
39
|
+
`pip install methplotlib`
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
## USAGE
|
|
44
|
+
```
|
|
45
|
+
methplotlib [-h] [-v] -m METHYLATION [METHYLATION ...] -n NAMES
|
|
46
|
+
[NAMES ...] -w WINDOW [-g GTF] [-b BED] [-f FASTA]
|
|
47
|
+
[--simplify] [--split] [--static STATIC] [--smooth SMOOTH]
|
|
48
|
+
[--dotsize DOTSIZE] [--example] [-o OUTFILE] [-q QCFILE]
|
|
49
|
+
|
|
50
|
+
plotting nanopolish methylation calls or frequency
|
|
51
|
+
|
|
52
|
+
optional arguments:
|
|
53
|
+
-h, --help show this help message and exit
|
|
54
|
+
-v, --version Print version and exit.
|
|
55
|
+
-m, --methylation METHYLATION [METHYLATION ...]
|
|
56
|
+
data in nanopolish, nanocompore, ont-cram or bedgraph
|
|
57
|
+
format
|
|
58
|
+
-n, --names NAMES [NAMES ...]
|
|
59
|
+
names of datasets in --methylation
|
|
60
|
+
-w, --window WINDOW window (region) to which the visualisation has to be restricted
|
|
61
|
+
-g, --gtf GTF add annotation based on a gtf file
|
|
62
|
+
-b, --bed BED add annotation based on a bed file
|
|
63
|
+
-f, --fasta FASTA required when --window is an entire chromosome, contig or transcript
|
|
64
|
+
--simplify simplify annotation track to show genes rather than transcripts
|
|
65
|
+
--split split, rather than overlay the methylation tracks
|
|
66
|
+
--static Make a static image of the browser window (filename)
|
|
67
|
+
--binary Make the nanopolish plot ignorning log likelihood nuances
|
|
68
|
+
--smooth Rolling window size for averaging frequency values (int)
|
|
69
|
+
--dotsize Control the size of dots in the per read plots (int)
|
|
70
|
+
--example Show example command and exit.
|
|
71
|
+
-o, --outfile OUTFILE File to write results to. Default:
|
|
72
|
+
methylation_browser_{chr}_{start}_{end}.html. Use
|
|
73
|
+
{region} as a shorthand for {chr}_{start}_{end} in the
|
|
74
|
+
filename. Missing paths will be created.
|
|
75
|
+
-q, --qcfile QCFILE File to write the qc report to. Default: The path in
|
|
76
|
+
outfile prefixed with qc_, default is qc_report_methyl
|
|
77
|
+
ation_browser_{chr}_{start}_{end}.html. Use {region}
|
|
78
|
+
as a shorthand for {chr}_{start}_{end} in the
|
|
79
|
+
filename. Missing paths will be created.
|
|
80
|
+
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Snakemake workflow
|
|
84
|
+
For streamlining nanopolish a Snakefile is included (using snakemake). The workflow uses a config file, of which an example is in this repository.
|
|
85
|
+
|
|
86
|
+
## Example data
|
|
87
|
+
The `examples` folder contains calls and frequencies for the human ACTB gene from PromethION sequencing of NA19240. An example command is available.
|
|
88
|
+
|
|
89
|
+
## Companion scripts
|
|
90
|
+
The `scripts` folder contains scripts for phasing modification calls in haplotypes based on [WhatsHap](https://whatshap.readthedocs.io/en/latest/) phasing, allele specific modification testing for phased data and differential modification testing across subjects.
|
|
91
|
+
|
|
92
|
+
## TO DO - CONTRIBUTIONS WELCOME
|
|
93
|
+
- Outlier detection (in windows) across samples
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
[](https://travis-ci.com/wdecoster/methplotlib)
|
|
2
|
-
[](https://conda.anaconda.org/bioconda)
|
|
3
1
|
[](https://anaconda.org/bioconda/methplotlib)
|
|
4
2
|
[](https://anaconda.org/bioconda/methplotlib)
|
|
5
3
|
|
|
@@ -8,12 +6,19 @@
|
|
|
8
6
|
This script generates a browser view on a window using data from
|
|
9
7
|
i) [nanopolish](https://github.com/jts/nanopolish), either as methylation calls or methylation frequencies (as processed by calculate_methylation_frequency.py). The methylation calls can additionally be phased using scripts/annotate_calls_by_phase.and scripts/split_calls_by_phase.py
|
|
10
8
|
ii) [nanocompore](https://github.com/tleonardi/nanocompore)
|
|
11
|
-
iii) in ont-cram format with MM/
|
|
9
|
+
iii) in ont-cram format with MM/ML tags according to the SAM specifications
|
|
12
10
|
iv) in bedgraph format
|
|
13
11
|
|
|
14
12
|
## INSTALLATION
|
|
13
|
+
|
|
14
|
+
Creating a new conda environment:
|
|
15
|
+
`conda create -n methplotlib methplotlib`
|
|
16
|
+
|
|
17
|
+
Or using pip:
|
|
15
18
|
`pip install methplotlib`
|
|
16
19
|
|
|
20
|
+
|
|
21
|
+
|
|
17
22
|
## USAGE
|
|
18
23
|
```
|
|
19
24
|
methplotlib [-h] [-v] -m METHYLATION [METHYLATION ...] -n NAMES
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
|
2
|
-
|Anaconda-Server Badge|
|
|
1
|
+
|Anaconda-Server Badge| |image1|
|
|
3
2
|
|
|
4
3
|
METHPLOTLIB
|
|
5
4
|
===========
|
|
@@ -11,15 +10,18 @@ METHPLOTLIB
|
|
|
11
10
|
additionally be phased using scripts/annotate_calls_by_phase.and
|
|
12
11
|
scripts/split_calls_by_phase.py
|
|
13
12
|
| ii) `nanocompore <https://github.com/tleonardi/nanocompore>`__
|
|
14
|
-
| iii) in ont-cram format with MM/
|
|
15
|
-
specifications
|
|
16
|
-
script <https://github.com/kpalin/gcf52ref/blob/f5_to_usam/scripts/extract_methylation_fast5_to_sam.py>`__
|
|
13
|
+
| iii) in ont-cram format with MM/ML tags according to the SAM
|
|
14
|
+
specifications
|
|
17
15
|
| iv) in bedgraph format
|
|
18
16
|
|
|
19
17
|
INSTALLATION
|
|
20
18
|
------------
|
|
21
19
|
|
|
22
|
-
|
|
20
|
+
| Creating a new conda environment:
|
|
21
|
+
| ``conda create -n methplotlib methplotlib``
|
|
22
|
+
|
|
23
|
+
| Or using pip:
|
|
24
|
+
| ``pip install methplotlib``
|
|
23
25
|
|
|
24
26
|
USAGE
|
|
25
27
|
-----
|
|
@@ -90,11 +92,7 @@ TO DO - CONTRIBUTIONS WELCOME
|
|
|
90
92
|
|
|
91
93
|
- Outlier detection (in windows) across samples
|
|
92
94
|
|
|
93
|
-
.. |Build Status| image:: https://travis-ci.com/wdecoster/methplotlib.svg?branch=master
|
|
94
|
-
:target: https://travis-ci.com/wdecoster/methplotlib
|
|
95
|
-
.. |Anaconda-Server Badge| image:: https://anaconda.org/bioconda/methplotlib/badges/installer/conda.svg
|
|
96
|
-
:target: https://conda.anaconda.org/bioconda
|
|
97
95
|
.. |Anaconda-Server Badge| image:: https://anaconda.org/bioconda/methplotlib/badges/version.svg
|
|
98
96
|
:target: https://anaconda.org/bioconda/methplotlib
|
|
99
|
-
.. |
|
|
97
|
+
.. |image1| image:: https://anaconda.org/bioconda/methplotlib/badges/license.svg
|
|
100
98
|
:target: https://anaconda.org/bioconda/methplotlib
|
|
@@ -0,0 +1,482 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import pyranges as pr
|
|
3
|
+
import numpy as np
|
|
4
|
+
import sys
|
|
5
|
+
import logging
|
|
6
|
+
from methplotlib.utils import file_sniffer, flatten
|
|
7
|
+
from itertools import repeat
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Modification(object):
|
|
11
|
+
def __init__(self, table, data_type, name, called_sites, start_end_table=None):
|
|
12
|
+
self.table = table
|
|
13
|
+
self.data_type = data_type
|
|
14
|
+
self.name = name
|
|
15
|
+
self.called_sites = called_sites
|
|
16
|
+
self.start_end_table = start_end_table
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def get_data(methylation_files, names, window, smoothen=5):
|
|
20
|
+
"""
|
|
21
|
+
Import methylation data from all files in the list methylation_files
|
|
22
|
+
|
|
23
|
+
Data can in various formats
|
|
24
|
+
- nanopolish raw, phased, frequency
|
|
25
|
+
- cram
|
|
26
|
+
- bam
|
|
27
|
+
- nanocompore
|
|
28
|
+
- bedgraph
|
|
29
|
+
- bedmethyl in the extended form (as obtained from modbam2bed --extended)
|
|
30
|
+
|
|
31
|
+
data is extracted within the window args.window
|
|
32
|
+
Frequencies are smoothened using a sliding window
|
|
33
|
+
"""
|
|
34
|
+
return flatten([read_mods(f, n, window, smoothen) for f, n in zip(methylation_files, names)])
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def read_mods(filename, name, window, smoothen=5):
|
|
38
|
+
"""
|
|
39
|
+
converts a file from nanopolish to a pandas dataframe
|
|
40
|
+
input can be from calculate_methylation_frequency
|
|
41
|
+
which will return a dataframe with 'chromosome', 'pos', 'methylated_frequency'
|
|
42
|
+
smoothening the result by a rolling average
|
|
43
|
+
|
|
44
|
+
input can also be raw data per read, optionally phased
|
|
45
|
+
which will return a dataframe with 'read', 'chromosome', 'pos', 'log_lik_ratio', 'strand'
|
|
46
|
+
"""
|
|
47
|
+
file_type = file_sniffer(filename)
|
|
48
|
+
logging.info(f"File {filename} is of type {file_type}")
|
|
49
|
+
try:
|
|
50
|
+
if file_type.startswith("nanopolish"):
|
|
51
|
+
return parse_nanopolish(filename, file_type, name, window, smoothen=smoothen)
|
|
52
|
+
elif file_type == "nanocompore":
|
|
53
|
+
return [parse_nanocompore(filename, name, window)]
|
|
54
|
+
elif file_type in ["cram", "bam"]:
|
|
55
|
+
return parse_cram(filename, file_type, name, window)
|
|
56
|
+
elif file_type == "bedgraph":
|
|
57
|
+
return [parse_bedgraph(filename, name, window)]
|
|
58
|
+
elif file_type == "bedmethyl_extended":
|
|
59
|
+
return [parse_bedmethyl(filename, name, window, smoothen=smoothen, flavor="modbam2bed")]
|
|
60
|
+
elif file_type == "bedmethyl":
|
|
61
|
+
return [parse_bedmethyl(filename, name, window, smoothen=smoothen, flavor="modkit")]
|
|
62
|
+
except Exception as e:
|
|
63
|
+
logging.error(f"Error processing {filename}.")
|
|
64
|
+
logging.error(e, exc_info=True)
|
|
65
|
+
sys.stderr.write(f"\n\n\nError processing {filename}!\n")
|
|
66
|
+
sys.stderr.write("\n\n\nDetailed error:\n")
|
|
67
|
+
raise
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def parse_nanopolish(filename, file_type, name, window, smoothen=5):
|
|
71
|
+
if window:
|
|
72
|
+
from pathlib import Path
|
|
73
|
+
|
|
74
|
+
if Path(filename + ".tbi").is_file():
|
|
75
|
+
import subprocess
|
|
76
|
+
import gzip
|
|
77
|
+
|
|
78
|
+
logging.info(f"Reading {filename} using a tabix stream.")
|
|
79
|
+
region = f"{window.chromosome}:{window.begin}-{window.end}"
|
|
80
|
+
try:
|
|
81
|
+
tabix_stream = subprocess.Popen(
|
|
82
|
+
["tabix", filename, region],
|
|
83
|
+
stdout=subprocess.PIPE,
|
|
84
|
+
stderr=subprocess.PIPE,
|
|
85
|
+
)
|
|
86
|
+
except FileNotFoundError as e:
|
|
87
|
+
logging.error("Error when opening a tabix stream.")
|
|
88
|
+
logging.error(e, exc_info=True)
|
|
89
|
+
sys.stderr.write("\n\nERROR when opening a tabix stream.\n")
|
|
90
|
+
sys.stderr.write("Is tabix installed and on the PATH?.")
|
|
91
|
+
raise
|
|
92
|
+
header = gzip.open(filename, "rt").readline().rstrip().split("\t")
|
|
93
|
+
table = pd.read_csv(tabix_stream.stdout, sep="\t", header=None, names=header)
|
|
94
|
+
else:
|
|
95
|
+
logging.info(f"Reading {filename} slowly by splitting the file in chunks.")
|
|
96
|
+
sys.stderr.write(f"\nReading {filename} would be faster with bgzip and tabix.\n")
|
|
97
|
+
if file_type in ["nanopolish_call", "nanopolish_phased"]:
|
|
98
|
+
sys.stderr.write("Please index with 'tabix -S1 -s1 -b3 -e4'.\n")
|
|
99
|
+
else:
|
|
100
|
+
sys.stderr.write("Please index with 'tabix -S1 -s1 -b2 -e3'.\n")
|
|
101
|
+
iter_csv = pd.read_csv(filename, sep="\t", iterator=True, chunksize=1e6)
|
|
102
|
+
table = pd.concat(
|
|
103
|
+
[chunk[chunk["chromosome"] == window.chromosome] for chunk in iter_csv]
|
|
104
|
+
)
|
|
105
|
+
else:
|
|
106
|
+
table = pd.read_csv(filename, sep="\t")
|
|
107
|
+
gr = pr.PyRanges(
|
|
108
|
+
table.rename(
|
|
109
|
+
columns={
|
|
110
|
+
"start": "Start",
|
|
111
|
+
"chromosome": "Chromosome",
|
|
112
|
+
"end": "End",
|
|
113
|
+
"Strand": "strand",
|
|
114
|
+
}
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
logging.info("Read the file in a dataframe.")
|
|
118
|
+
|
|
119
|
+
if window:
|
|
120
|
+
gr = gr[str(window.chromosome), window.begin : window.end]
|
|
121
|
+
try:
|
|
122
|
+
gr.pos = np.floor(gr.drop().df[["Start", "End"]].mean(axis=1))
|
|
123
|
+
except KeyError:
|
|
124
|
+
sys.stderr.write(f"\n\n\nProblem parsing nanopolish file {filename}!\n")
|
|
125
|
+
sys.stderr.write(
|
|
126
|
+
"Could it be that there are no calls in your selected window "
|
|
127
|
+
f"{window.chromosome}:{window.begin}-{window.end}?\n"
|
|
128
|
+
)
|
|
129
|
+
sys.stderr.write("\n\n\nDetailed error:\n")
|
|
130
|
+
raise
|
|
131
|
+
|
|
132
|
+
table = gr.df
|
|
133
|
+
|
|
134
|
+
if file_type in ["nanopolish_call", "nanopolish_phased"]:
|
|
135
|
+
table = table.drop(
|
|
136
|
+
columns=[
|
|
137
|
+
"Start",
|
|
138
|
+
"End",
|
|
139
|
+
"log_lik_methylated",
|
|
140
|
+
"log_lik_unmethylated",
|
|
141
|
+
"num_calling_strands",
|
|
142
|
+
"num_motifs",
|
|
143
|
+
"sequence",
|
|
144
|
+
]
|
|
145
|
+
)
|
|
146
|
+
if "motif" in table:
|
|
147
|
+
return [
|
|
148
|
+
Modification(
|
|
149
|
+
table=sub_df,
|
|
150
|
+
data_type=file_type,
|
|
151
|
+
name=f"{name}_{mod}",
|
|
152
|
+
called_sites=len(sub_df),
|
|
153
|
+
)
|
|
154
|
+
for mod, sub_df in table.groupby("motif")
|
|
155
|
+
]
|
|
156
|
+
else:
|
|
157
|
+
return [
|
|
158
|
+
Modification(
|
|
159
|
+
table=table.sort_values(["read_name", "pos"]),
|
|
160
|
+
data_type=file_type,
|
|
161
|
+
name=name,
|
|
162
|
+
called_sites=len(table),
|
|
163
|
+
)
|
|
164
|
+
]
|
|
165
|
+
if file_type == "nanopolish_freq":
|
|
166
|
+
called_sites = table.called_sites
|
|
167
|
+
chromosome = table.Chromosome.values[0]
|
|
168
|
+
|
|
169
|
+
table = table.drop(
|
|
170
|
+
columns=[
|
|
171
|
+
"Chromosome",
|
|
172
|
+
"Start",
|
|
173
|
+
"End",
|
|
174
|
+
"num_motifs_in_group",
|
|
175
|
+
"called_sites",
|
|
176
|
+
"called_sites_methylated",
|
|
177
|
+
"group_sequence",
|
|
178
|
+
]
|
|
179
|
+
)
|
|
180
|
+
return [
|
|
181
|
+
Modification(
|
|
182
|
+
table=table.sort_values("pos")
|
|
183
|
+
.groupby("pos")
|
|
184
|
+
.mean()
|
|
185
|
+
.rolling(window=smoothen, center=True)
|
|
186
|
+
.mean()
|
|
187
|
+
.assign(Chromosome=chromosome),
|
|
188
|
+
data_type=file_type,
|
|
189
|
+
name=name,
|
|
190
|
+
called_sites=called_sites.sum(),
|
|
191
|
+
)
|
|
192
|
+
]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def parse_nanocompore(filename, name, window):
|
|
196
|
+
def nanocompore_columns_of_interest(column):
|
|
197
|
+
if column in ["pos", "ref_id"]:
|
|
198
|
+
return True
|
|
199
|
+
elif column.endswith("pvalue_context_2") or column.endswith("pvalue"):
|
|
200
|
+
return True
|
|
201
|
+
else:
|
|
202
|
+
return False
|
|
203
|
+
|
|
204
|
+
table = pd.read_csv(filename, sep="\t", usecols=nanocompore_columns_of_interest)
|
|
205
|
+
if window:
|
|
206
|
+
table = table[table["ref_id"] == window.chromosome]
|
|
207
|
+
return Modification(
|
|
208
|
+
table=table.sort_values("pos")
|
|
209
|
+
.append({"pos": window.end}, ignore_index=True)
|
|
210
|
+
.drop(columns="ref_id")
|
|
211
|
+
.fillna(1.0),
|
|
212
|
+
data_type="nanocompore",
|
|
213
|
+
name=name,
|
|
214
|
+
called_sites=len(table),
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def parse_bedgraph(filename, name, window):
|
|
219
|
+
if window:
|
|
220
|
+
from pathlib import Path
|
|
221
|
+
|
|
222
|
+
if Path(filename + ".tbi").is_file():
|
|
223
|
+
import subprocess
|
|
224
|
+
|
|
225
|
+
logging.info(f"Reading {filename} using a tabix stream.")
|
|
226
|
+
region = f"{window.chromosome}:{window.begin}-{window.end}"
|
|
227
|
+
try:
|
|
228
|
+
tabix_stream = subprocess.Popen(
|
|
229
|
+
["tabix", filename, region],
|
|
230
|
+
stdout=subprocess.PIPE,
|
|
231
|
+
stderr=subprocess.PIPE,
|
|
232
|
+
)
|
|
233
|
+
except FileNotFoundError as e:
|
|
234
|
+
logging.error("Error when opening a tabix stream.")
|
|
235
|
+
logging.error(e, exc_info=True)
|
|
236
|
+
sys.stderr.write("\n\nERROR when opening a tabix stream.\n")
|
|
237
|
+
sys.stderr.write("Is tabix installed and on the PATH?.")
|
|
238
|
+
raise
|
|
239
|
+
table = pd.read_csv(
|
|
240
|
+
tabix_stream.stdout,
|
|
241
|
+
sep="\t",
|
|
242
|
+
header=None,
|
|
243
|
+
names=["Chromosome", "Start", "End", "Value"],
|
|
244
|
+
)
|
|
245
|
+
else:
|
|
246
|
+
logging.info(f"Reading {filename} slowly by splitting the file in chunks.")
|
|
247
|
+
sys.stderr.write(
|
|
248
|
+
f"\nReading {filename} would be faster with bgzip and 'tabix -p bed'.\n"
|
|
249
|
+
)
|
|
250
|
+
iter_csv = pd.read_csv(
|
|
251
|
+
filename,
|
|
252
|
+
sep="\t",
|
|
253
|
+
iterator=True,
|
|
254
|
+
chunksize=1e6,
|
|
255
|
+
header=None,
|
|
256
|
+
names=["Chromosome", "Start", "End", "Value"],
|
|
257
|
+
)
|
|
258
|
+
table = pd.concat(
|
|
259
|
+
[chunk[chunk["Chromosome"] == window.chromosome] for chunk in iter_csv]
|
|
260
|
+
)
|
|
261
|
+
else:
|
|
262
|
+
table = pd.read_csv(
|
|
263
|
+
filename,
|
|
264
|
+
sep="\t",
|
|
265
|
+
header=None,
|
|
266
|
+
names=["Chromosome", "Start", "End", "Value"],
|
|
267
|
+
)
|
|
268
|
+
gr = pr.PyRanges(table)
|
|
269
|
+
logging.info("Read the file in a dataframe.")
|
|
270
|
+
if window:
|
|
271
|
+
gr = gr[window.chromosome, window.begin : window.end]
|
|
272
|
+
if len(gr.df) == 0:
|
|
273
|
+
sys.exit(f"No records for {filename} in {window.string}!\n")
|
|
274
|
+
return Modification(
|
|
275
|
+
table=gr.df.sort_values("Start"),
|
|
276
|
+
data_type="bedgraph",
|
|
277
|
+
name=name,
|
|
278
|
+
called_sites=len(table),
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def parse_bedmethyl(filename, name, window, smoothen=5, flavor="modkit"):
|
|
283
|
+
if not flavor in ["modkit", "modbam2bed"]:
|
|
284
|
+
sys.exit(f"ERROR: flavor {flavor} not supported for bedmethyl.")
|
|
285
|
+
if flavor == "modkit":
|
|
286
|
+
colnames = {
|
|
287
|
+
0: "Chromosome",
|
|
288
|
+
1: "Start",
|
|
289
|
+
2: "End",
|
|
290
|
+
# if multiple types of modification should be supported additionally column 3 has to be read
|
|
291
|
+
# 3: "Modification",
|
|
292
|
+
10: "Frequency",
|
|
293
|
+
}
|
|
294
|
+
else:
|
|
295
|
+
colnames = {
|
|
296
|
+
0: "Chromosome",
|
|
297
|
+
1: "Start",
|
|
298
|
+
2: "End",
|
|
299
|
+
# if multiple types of modification should be supported additionally column 3 has to be read
|
|
300
|
+
# 3: "Modification",
|
|
301
|
+
11: "canonical",
|
|
302
|
+
12: "modified",
|
|
303
|
+
}
|
|
304
|
+
usecols = colnames.keys()
|
|
305
|
+
if window:
|
|
306
|
+
from pathlib import Path
|
|
307
|
+
|
|
308
|
+
if Path(filename + ".tbi").is_file():
|
|
309
|
+
import subprocess
|
|
310
|
+
|
|
311
|
+
logging.info(f"Reading {filename} using a tabix stream.")
|
|
312
|
+
region = f"{window.chromosome}:{window.begin}-{window.end}"
|
|
313
|
+
try:
|
|
314
|
+
tabix_stream = subprocess.Popen(
|
|
315
|
+
["tabix", filename, region],
|
|
316
|
+
stdout=subprocess.PIPE,
|
|
317
|
+
stderr=subprocess.PIPE,
|
|
318
|
+
)
|
|
319
|
+
except FileNotFoundError as e:
|
|
320
|
+
logging.error("Error when opening a tabix stream.")
|
|
321
|
+
logging.error(e, exc_info=True)
|
|
322
|
+
sys.stderr.write("\n\nERROR when opening a tabix stream.\n")
|
|
323
|
+
sys.stderr.write("Is tabix installed and on the PATH?.")
|
|
324
|
+
raise
|
|
325
|
+
table = pd.read_csv(
|
|
326
|
+
tabix_stream.stdout,
|
|
327
|
+
sep="\t",
|
|
328
|
+
header=None,
|
|
329
|
+
usecols=usecols,
|
|
330
|
+
).rename(columns=colnames)
|
|
331
|
+
else:
|
|
332
|
+
logging.info(f"Reading {filename} slowly by splitting the file in chunks.")
|
|
333
|
+
sys.stderr.write(
|
|
334
|
+
f"\nReading {filename} would be faster with bgzip and 'tabix -p bed'.\n"
|
|
335
|
+
)
|
|
336
|
+
iter_csv = pd.read_csv(
|
|
337
|
+
filename,
|
|
338
|
+
sep="\t",
|
|
339
|
+
iterator=True,
|
|
340
|
+
chunksize=1e6,
|
|
341
|
+
header=None,
|
|
342
|
+
usecols=usecols,
|
|
343
|
+
)
|
|
344
|
+
table = pd.concat([chunk[chunk[0] == window.chromosome] for chunk in iter_csv]).rename(
|
|
345
|
+
columns=colnames
|
|
346
|
+
)
|
|
347
|
+
else:
|
|
348
|
+
table = pd.read_csv(
|
|
349
|
+
filename,
|
|
350
|
+
sep="\t",
|
|
351
|
+
header=None,
|
|
352
|
+
usecols=usecols,
|
|
353
|
+
).rename(columns=colnames)
|
|
354
|
+
gr = pr.PyRanges(table)
|
|
355
|
+
logging.info("Read the file in a dataframe.")
|
|
356
|
+
if window:
|
|
357
|
+
gr = gr[window.chromosome, window.begin : window.end]
|
|
358
|
+
if len(gr.df) == 0:
|
|
359
|
+
sys.exit(f"No records for {filename} in {window.string}!\n")
|
|
360
|
+
table = gr.df.sort_values("Start")
|
|
361
|
+
if flavor == "modkit":
|
|
362
|
+
table["modified_frequency"] = table["Frequency"] / 100
|
|
363
|
+
table.drop(columns=["Frequency"], inplace=True)
|
|
364
|
+
else:
|
|
365
|
+
table["modified_frequency"] = table["modified"] / (table["canonical"] + table["modified"])
|
|
366
|
+
table.drop(
|
|
367
|
+
columns=["Chromosome", "End", "canonical", "modified"], inplace=True, errors="ignore"
|
|
368
|
+
)
|
|
369
|
+
return Modification(
|
|
370
|
+
table=table.rolling(window=smoothen, center=True).mean(),
|
|
371
|
+
data_type="bedmethyl_extended",
|
|
372
|
+
name=name,
|
|
373
|
+
called_sites=len(table),
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def parse_cram(filename, filetype, name, window):
|
|
378
|
+
import pysam
|
|
379
|
+
|
|
380
|
+
mode = "rc" if filetype == "cram" else "rb"
|
|
381
|
+
cram = pysam.AlignmentFile(filename, mode)
|
|
382
|
+
data = []
|
|
383
|
+
start_stops = []
|
|
384
|
+
for read in cram.fetch(reference=str(window.chromosome), start=window.begin, end=window.end):
|
|
385
|
+
if not read.is_supplementary and not read.is_secondary:
|
|
386
|
+
start_stops.append((read.query_name, read.reference_start, read.reference_end))
|
|
387
|
+
mod_positions = get_modified_reference_positions(read)
|
|
388
|
+
if mod_positions:
|
|
389
|
+
data.extend(mod_positions)
|
|
390
|
+
df = (
|
|
391
|
+
pd.DataFrame(data, columns=["read_name", "strand", "pos", "quality", "mod"])
|
|
392
|
+
.astype(dtype={"mod": "category", "quality": "float"})
|
|
393
|
+
.sort_values(["read_name", "pos"])
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
return [
|
|
397
|
+
Modification(
|
|
398
|
+
table=sub_df,
|
|
399
|
+
data_type="ont-cram",
|
|
400
|
+
name=f"{name}_{mod}",
|
|
401
|
+
called_sites=len(sub_df),
|
|
402
|
+
start_end_table=pd.DataFrame(
|
|
403
|
+
start_stops, columns=["read_name", "posmin", "posmax"]
|
|
404
|
+
).set_index("read_name"),
|
|
405
|
+
)
|
|
406
|
+
for mod, sub_df in df.groupby("mod")
|
|
407
|
+
]
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def get_modified_reference_positions(read):
|
|
411
|
+
mod_positions = []
|
|
412
|
+
if read.has_tag("Mm") or read.has_tag("MM"):
|
|
413
|
+
modtag = "Mm" if read.has_tag("Mm") else "MM"
|
|
414
|
+
qualtag = "Ml" if read.has_tag("Ml") else "ML"
|
|
415
|
+
offset = 0
|
|
416
|
+
for context in read.get_tag(modtag).split(";"):
|
|
417
|
+
if context:
|
|
418
|
+
basemod = context.split(",", 1)[0]
|
|
419
|
+
if "-" in basemod:
|
|
420
|
+
sys.exit(
|
|
421
|
+
"ERROR: modifications on negative strand currently unsupported.\n"
|
|
422
|
+
"Please contact me if this would be of interest for you."
|
|
423
|
+
)
|
|
424
|
+
base, mod = basemod.split("+")
|
|
425
|
+
# code below does not work with for ambiguous N and will search for a literal N
|
|
426
|
+
if base == "N":
|
|
427
|
+
sys.exit(
|
|
428
|
+
"ERROR: modifications of N nucleotides currently unsupported.\n"
|
|
429
|
+
"Please contact me if this would be of interest for you."
|
|
430
|
+
)
|
|
431
|
+
# The positions are encoded by specifying the number of non-modified occurences
|
|
432
|
+
# of that specific bases to skip in the read sequence
|
|
433
|
+
deltas = [int(i) for i in context.split(",")[1:]]
|
|
434
|
+
if not deltas:
|
|
435
|
+
continue
|
|
436
|
+
# Make an array with all positions for which a modification is reported
|
|
437
|
+
# In coordinates relative to occurences of the nucleotide in the read
|
|
438
|
+
locations = np.cumsum(deltas) + np.arange(len(deltas))
|
|
439
|
+
# Nake an array with all read positions for which that nucleotide exists in the read
|
|
440
|
+
# Oddly, get_forward_sequence() returns the sequence as it came from the sequencer
|
|
441
|
+
# This is the same direction as the Mm/Ml tags are specified in
|
|
442
|
+
base_index = np.array(
|
|
443
|
+
[i for i, letter in enumerate(read.get_forward_sequence()) if letter == base]
|
|
444
|
+
)
|
|
445
|
+
# Based on both arrays, find those read coordinates that have a modification
|
|
446
|
+
modified_bases = base_index[locations]
|
|
447
|
+
# Convert the array of read coordinates to reference coordinates
|
|
448
|
+
# full_length=True will make sure the list is of the same length as the read
|
|
449
|
+
# by adding 'None' for softclipped nucleotides
|
|
450
|
+
refpos = np.array(read.get_reference_positions(full_length=True))
|
|
451
|
+
# read.get_reference_positions() returns incrementing coordinates
|
|
452
|
+
# regardless of strand, so reversing the order for reverse_stranded reads
|
|
453
|
+
if read.is_reverse:
|
|
454
|
+
refpos = refpos[::-1]
|
|
455
|
+
# The likelihoods are in an array of length of all Mm/MM deltas,
|
|
456
|
+
# and are not separated by context/modified nucleotide type
|
|
457
|
+
if read.has_tag(qualtag):
|
|
458
|
+
likelihoods = read.get_tag(qualtag).tolist()[offset : offset + len(deltas)]
|
|
459
|
+
else:
|
|
460
|
+
likelihoods = [255] * len(deltas)
|
|
461
|
+
offset += len(deltas)
|
|
462
|
+
mod_positions.extend(
|
|
463
|
+
zip(
|
|
464
|
+
repeat(read.query_name), # read_name
|
|
465
|
+
repeat("-" if read.is_reverse else "+"), # strand
|
|
466
|
+
refpos[modified_bases], # pos
|
|
467
|
+
likelihoods, # quality
|
|
468
|
+
repeat(basemod), # mod
|
|
469
|
+
)
|
|
470
|
+
)
|
|
471
|
+
return mod_positions
|
|
472
|
+
else:
|
|
473
|
+
return None
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def errs_tab(n):
|
|
477
|
+
"""Generate list of error rates for qualities less than equal than n."""
|
|
478
|
+
return [10 ** (q / -10) for q in range(n + 1)]
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def phred_to_probability(quals, tab=errs_tab(128)):
|
|
482
|
+
return [tab[ord(q) - 33] for q in quals]
|