methplotlib 0.23.0__py3-none-any.whl

methplotlib/qc.py

@@ -0,0 +1,133 @@
+from sklearn.decomposition import PCA
+import plotly.express as px
+import pandas as pd
+import plotly
+import plotly.graph_objs as go
+
+
+def qc_plots(meth_data, window, qcpath=None, outpath=None):
+
+    if qcpath is None and outpath is None:
+        outfile = f"qc_report_{window.string}.html"
+    elif qcpath is None:
+        from pathlib import Path, PosixPath
+        p = Path(outpath.format(region=window.string))
+        Path.mkdir(p.parent, exist_ok=True, parents=True)
+        outfile = str(p.parent / PosixPath("qc_" + p.stem + ".html"))
+    else:
+        from pathlib import Path
+        p = Path(qcpath)
+        Path.mkdir(p.parent, exist_ok=True, parents=True)
+        outfile = qcpath
+
+    with open(outfile, 'w') as qc_report:
+        qc_report.write(num_sites_bar(meth_data))
+        if len([m for m in meth_data if m.data_type == "nanopolish_freq"]) > 0:
+            # only the frequency column, as the Chromosome column of every dataset
+            # would collide in the join and end up in the plots below
+            data = [m.table[["methylated_frequency"]].rename({"methylated_frequency": m.name},
+                                                             axis='columns')
+                    for m in meth_data if m.data_type == "nanopolish_freq"]
+            full = data[0].join(data[1:]).dropna(how="any", axis="index")
+            qc_report.write(modified_fraction_histogram(full))
+        if len([m for m in meth_data if m.data_type == "nanopolish_freq"]) > 2:
+            qc_report.write(pairwise_correlation_plot(full))
+            qc_report.write(pca(full))
+            qc_report.write(global_box(data))
+        if len([m for m in meth_data
+                if m.data_type in ["nanopolish_call", "nanopolish_phased"]]) > 2:
+            pass
+
+
+def num_sites_bar(meth_data):
+    trace = go.Bar(x=[m.name for m in meth_data],
+                   y=[m.called_sites for m in meth_data])
+    layout = dict(title="Number of called positions")
+    return plotly.offline.plot(dict(data=[trace], layout=layout),
+                               output_type="div",
+                               show_link=False,
+                               include_plotlyjs='cdn')
+
+
+def pairwise_correlation_plot(full):
+    trace = go.Splom(dimensions=[dict(label=l, values=full[l]) for l in full.columns],
+                     marker=dict(size=4,
+                                 line=dict(width=0.5,
+                                           color='rgb(230,230,230)')),
+                     diagonal=dict(visible=False)
+                     )
+
+    layout = go.Layout(
+        title='Correlation of modification frequency',
+        dragmode='select',
+        width=1200,
+        height=1200,
+        autosize=False,
+        hovermode=False,
+        plot_bgcolor='rgba(240,240,240, 0.95)')
+
+    for i in range(1, len(full.columns) + 1):
+        layout[f"xaxis{i}"] = dict(showline=True, zeroline=False, gridcolor='#fff', ticklen=4)
+        layout[f"yaxis{i}"] = dict(showline=True, zeroline=False, gridcolor='#fff', ticklen=4)
+
+    return plotly.offline.plot(dict(data=[trace], layout=layout),
+                               output_type="div",
+                               show_link=False,
+                               include_plotlyjs='cdn')
+
+
+def pca(full):
+    sklearn_pca = PCA(n_components=2)
+    pca = sklearn_pca.fit_transform(full.transpose())
+    data = [dict(type='scatter',
+                 x=[pca[index, 0]],
+                 y=[pca[index, 1]],
+                 mode='markers',
+                 name=name,
+                 hoverinfo='name',
+                 marker=dict(
+                     size=12,
+                     line=dict(
+                         color='rgba(217, 217, 217, 0.14)',
+                         width=0.5),
+                     opacity=0.8))
+            for index, name in enumerate(full.columns)]
+
+    layout = dict(xaxis=dict(title='PC1', showline=False),
+                  yaxis=dict(title='PC2', showline=False),
+                  title="Principal Component Analysis"
+                  )
+    return plotly.offline.plot(dict(data=data, layout=layout),
+                               output_type="div",
+                               show_link=False,
+                               include_plotlyjs='cdn')
+
+
+def global_box(data):
+    fig = px.box(pd.concat([d.reset_index(drop=True)
+                            .rename({d.columns[0]: "freq"}, axis="columns")
+                            .assign(dataset=d.columns[0]) for d in data], ignore_index=True),
+                 x="dataset", y="freq", title="Global frequency of modification")
+    return plotly.offline.plot(fig,
+                               output_type="div",
+                               show_link=False,
+                               include_plotlyjs='cdn')
+
+
+def modified_fraction_histogram(full):
+    traces = [go.Histogram(x=full[dataset].dropna(),
+                           histnorm='probability density',
+                           xbins=dict(start=0, size=0.01, end=1),
+                           name=dataset,
+                           opacity=0.6
+                           )
+              for dataset in full.columns]
+    layout = dict(barmode="overlay",
+                  title="Histogram of modified fractions",
+                  xaxis=dict(title="Modified fraction"),
+                  yaxis=dict(title="Frequency"))
+    return plotly.offline.plot(dict(data=traces,
+                                    layout=layout),
+                               output_type="div",
+                               show_link=False,
+                               include_plotlyjs='cdn')

methplotlib/utils.py

@@ -0,0 +1,409 @@
+from argparse import ArgumentParser, SUPPRESS
+import sys
+from math import ceil
+from methplotlib.version import __version__
+from datetime import datetime as dt
+from time import time
+import logging
+import plotly
+from pathlib import Path
+from itertools import chain
+from plotly import subplots
+from plotly.subplots import make_subplots
+
+
+class Region(object):
+    def __init__(self, region, fasta=None):
+        if ":" in region:
+            try:
+                # the chromosome is deliberately kept as a string,
+                # as that is how pyranges and tabix identify chromosomes
+                self.chromosome, interval = region.replace(",", "").split(":")
+                self.begin, self.end = [int(i) for i in interval.split("-")]
+                self.start = self.begin
+            except ValueError:
+                sys.exit(
+                    "\n\nERROR: Window (-w/--window) inproperly formatted, "
+                    "examples of accepted formats are:\n"
+                    "'chr5:150200605-150423790' or 'ENST00000647408'\n\n"
+                )
+            self.size = self.end - self.begin
+            if not self.size > 0:
+                sys.exit(
+                    "\n\nERROR: Window (-w/--window) inproperly formatted, "
+                    "begin of the interval has to be smaller than end\n\n"
+                )
+            self.string = f"{self.chromosome}_{self.begin}_{self.end}"
+        else:  # When region is an entire chromosome, contig or transcript
+            if fasta is None:
+                sys.exit(
+                    "A fasta reference file is required if --window "
+                    "is an entire chromosome, contig or transcript"
+                )
+            else:
+                from pyfaidx import Fasta
+
+                self.chromosome = region
+                self.begin = 0
+                self.string = region
+                self.end = len(Fasta(fasta)[region])
+                self.size = self.end
+
+
+def make_windows(full_window, max_size=1e6, fasta=None):
+    reg = Region(full_window, fasta)
+    if reg.size > max_size:
+        chunks = ceil(reg.size / max_size)
+        chsize = ceil(reg.size / chunks)
+        return [
+            Region(f"{reg.chromosome}:{reg.begin + i * chsize}-{reg.begin + (i + 1) * chsize}")
+            for i in range(chunks)
+        ]
+    else:
+        return [reg]
+
+
+def flatten(nested_list):
+    return list(chain.from_iterable(nested_list))
+
+
+def get_args():
+    parser = ArgumentParser(description="plotting nanopolish methylation calls or frequency")
+    parser.add_argument(
+        "-v",
+        "--version",
+        help="Print version and exit.",
+        action="version",
+        version=f"methplotlib {__version__}",
+    )
+    parser.add_argument(
+        "-m",
+        "--methylation",
+        nargs="+",
+        help="data in nanopolish, nanocompore, bam/cram with MM/ML tags, bedgraph or bedmethyl format",
+        required=True if "--example" not in sys.argv else False,
+    )
+    parser.add_argument(
+        "-n",
+        "--names",
+        nargs="+",
+        help="names of datasets in --methylation",
+        required=True if "--example" not in sys.argv else False,
+    )
+    parser.add_argument(
+        "-w",
+        "--window",
+        help="window (region) to which the visualisation has to be restricted",
+        required=True if "--example" not in sys.argv else False,
+    )
+    parser.add_argument("-g", "--gtf", help="add annotation based on a gtf file")
+    parser.add_argument("-b", "--bed", help="add annotation based on a bed file")
+    parser.add_argument(
+        "-f",
+        "--fasta",
+        help="required when --window is an entire chromosome, contig or transcript",
+    )
+    parser.add_argument(
+        "--simplify",
+        help="simplify annotation track to show genes rather than transcripts",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--split",
+        help="split, rather than overlay the methylation tracks",
+        action="store_true",
+    )
+    parser.add_argument("--static", help="Make a static image of the browser window")
+    parser.add_argument(
+        "--binary",
+        help="Make the nanopolish plot ignorning log likelihood nuances",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--smooth",
+        help="Rolling window size for averaging frequency values. "
+        "Counted in records, and both strands are separate records in bedmethyl",
+        type=int,
+        default=5,
+    )
+    parser.add_argument(
+        "--mods",
+        help="Comma separated list of modifications to restrict to",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--dotsize",
+        help="Control the size of dots in the per read plots",
+        type=int,
+        default=4,
+    )
+    parser.add_argument(
+        "--minqual",
+        help="The minimal modification probability to show, on the 0-255 scale "
+        "of the ML tag [for bam/cram input only]",
+        type=int,
+        default=20,
+    )
+    parser.add_argument("--example", action="store_true", help="Show example command and exit.")
+    parser.add_argument(
+        "-o",
+        "--outfile",
+        help="File to write results to. "
+        "Default: methylation_browser_{chr}_{start}_{end}.html. "
+        "Use {region} as a shorthand for {chr}_{start}_{end} in the filename. "
+        "Missing paths will be created.",
+    )
+    parser.add_argument(
+        "-q",
+        "--qcfile",
+        help="File to write the qc report to. "
+        "Default: The path in outfile prefixed with qc_, "
+        "default is qc_report_methylation_browser_{chr}_{start}_{end}.html. "
+        "Use {region} as a shorthand for {chr}_{start}_{end} in the filename. "
+        "Missing paths will be created.",
+    )
+    parser.add_argument("--store", help=SUPPRESS, action="store_true")
+    args = parser.parse_args()
+    if not args.example and not len(args.names) == len(args.methylation):
+        sys.exit("INPUT ERROR: Expecting the same number of names as datasets!")
+    return args
+
+
+def init_logs(args):
+    """Initiate log file and log arguments."""
+    start_time = dt.fromtimestamp(time()).strftime("%Y%m%d_%H%M")
+    logname = "methplotlib_" + start_time + ".log"
+    handlers = [logging.FileHandler(logname)]
+    logging.basicConfig(format="%(asctime)s %(message)s", handlers=handlers, level=logging.INFO)
+    logging.info(f"methplotlib {__version__} started.")
+    py_version = sys.version.replace("\n", " ")
+    logging.info(f"Python version is: {py_version}")
+    logging.info(f"Arguments are: {args}")
+
+
+def print_example():
+    """
+    Print an example command
+
+    The example files are not shipped with the package, to keep it small,
+    so the paths are relative to a clone of the repository
+    """
+    example = """
+methplotlib -m examples/NA19240-methylation_ACTB_calls.tsv.gz \\
+               examples/NA19240-methylation_ACTB_frequency.tsv.gz \\
+            -n calls frequencies \\
+            -w chr7:5,525,542-5,543,028 \\
+            -g examples/GRCh38-ACTB-locus.gtf.gz \\
+            --simplify \\
+            -b examples/DNase_cluster_ACTB.bed.gz \\
+            -o '{region}/example.html'""".strip()
+    sys.stderr.write(
+        "Example command, using the files in the examples folder of\n"
+        "https://github.com/wdecoster/methplotlib\n\n"
+    )
+    print(example)
+    sys.exit(0)
+
+
+def is_gz_file(filepath):
+    import binascii
+
+    with open(filepath, "rb") as test_f:
+        return binascii.hexlify(test_f.read(2)) == b"1f8b"
+
+
+def is_cram_file(filepath):
+    with open(filepath, "rb") as test_f:
+        return test_f.read(4) == b"CRAM"
+
+
+def is_bam_file(filepath):
+    import gzip
+
+    try:
+        with gzip.open(filepath) as test_f:
+            return test_f.read(3) == b"BAM"
+    except OSError:
+        return False
+
+
+def open_text(filepath):
+    """Open a file for reading text, transparently handling gzip compression."""
+    if is_gz_file(filepath):
+        import gzip
+
+        return gzip.open(filepath, "rt")
+    else:
+        return open(filepath, "r")
+
+
+def first_data_line(filename):
+    """
+    Return the first line of a file that contains data,
+    skipping empty lines, comments and browser/track lines
+    """
+    with open_text(filename) as handle:
+        for line in handle:
+            if line.strip() and not line.startswith(("#", "track", "browser")):
+                return line.rstrip("\n").rstrip("\r")
+    sys.exit(f"\n\nERROR: File {filename} does not contain any data!\n")
+
+
+def is_bedmethyl(fields):
+    """
+    Check if the fields of a line look like bedMethyl,
+    which is BED9 with a modification code in column 4, followed by counts
+    """
+    return (
+        len(fields) > 9
+        and fields[1].isdigit()
+        and fields[2].isdigit()
+        and fields[5] in ["+", "-", "."]
+    )
+
+
+def is_bedgraph(fields):
+    """
+    Check if the fields of a line look like bedgraph:
+    chromosome, start, end and a numeric value.
+    A fifth column (coverage, as written by `modkit pileup --bedgraph`) is tolerated.
+    """
+    if len(fields) not in [4, 5] or not (fields[1].isdigit() and fields[2].isdigit()):
+        return False
+    try:
+        float(fields[3])
+        return True
+    except ValueError:
+        return False
+
+
+def bedmethyl_separator(filename):
+    """
+    `modkit pileup` writes the first 9 (BED) columns tab separated
+    and the remaining columns space separated, unless --only-tabs was used.
+    Return the separator to read the file with.
+    """
+    line = first_data_line(filename)
+    return "\t" if len(line.split("\t")) == len(line.split()) else r"\s+"
+
+
+def unrecognized_file_error(filename, line):
+    """Compose an error message explaining why a file could not be used."""
+    fields = line.split("\t")
+    whitespace_fields = line.split()
+    return (
+        f"\n\nERROR: Input file {filename} was not recognized!\n\n"
+        f"Its first data line has {len(fields)} tab separated fields "
+        f"({len(whitespace_fields)} when splitting on any whitespace):\n"
+        f"{line[:200]}\n\n"
+        "methplotlib accepts:\n"
+        " - bedgraph: 4 columns (chromosome, start, end, value), "
+        "or the 5 columns of `modkit pileup --bedgraph`\n"
+        " - bedMethyl of `modkit pileup`: 18 columns, with or without --only-tabs\n"
+        " - bedMethyl of `modbam2bed --extended`: 14 columns\n"
+        " - bam or cram with MM/ML tags\n"
+        " - nanopolish call-methylation output, optionally phased\n"
+        " - the output of nanopolish/calculate_methylation_frequency.py\n"
+        " - nanocompore results\n\n"
+        "Frequent causes are a header line, a file that is comma or space separated "
+        "throughout, or extra columns added by other tools.\n"
+        "If your file should be supported, please open an issue at "
+        "https://github.com/wdecoster/methplotlib/issues and share these lines.\n"
+    )
+
+
+def file_sniffer(filename):
+    """
+    Takes in a filename and tries to guess the input file type
+    """
+    if not Path(filename).is_file():
+        sys.exit(f"\n\nERROR: File {filename} does not exist, please check the path!\n")
+    if is_cram_file(filename):
+        return "cram"
+    if is_bam_file(filename):
+        return "bam"
+    header = first_data_line(filename)
+
+    # nanocompore renamed GMM_anova_pvalue to GMM_logit_pvalue
+    if "GMM_anova_pvalue" in header or "GMM_logit_pvalue" in header:
+        return "nanocompore"
+    if "log_lik_methylated" in header:
+        if "PS" in header:
+            return "nanopolish_phased"
+        else:
+            return "nanopolish_call"
+    if "num_motifs_in_group" in header:
+        return "nanopolish_freq"
+    # bedMethyl files have no header and are tab separated,
+    # except for modkit which separates the columns after BED9 by spaces
+    # unless it was run with --only-tabs
+    whitespace_fields = header.split()
+    if is_bedmethyl(whitespace_fields):
+        if len(whitespace_fields) == 14:  # file generated by `modbam2bed --extended`
+            return "bedmethyl_extended"
+        if len(whitespace_fields) == 18:  # file generated by `modkit pileup`
+            return "bedmethyl"
+    if is_bedgraph(header.split("\t")):
+        return "bedgraph"
+    sys.exit(unrecognized_file_error(filename, header))
+
+
+def create_subplots(num_methrows, split, names=None, annotation=True):
+    """
+    Prepare the panels (rows * 1 column) for the subplots.
+    If splitting: one row for each dataset, taking 90%/len(datasets) for heights
+    If not: one row spanning 4 rows and taking 90% of the heights
+    if annotation is True (bed or gtf) then add a row with height 10%
+    """
+    if split:
+        return plotly.subplots.make_subplots(
+            rows=num_methrows + annotation,
+            cols=1,
+            shared_xaxes=True,
+            specs=[[{}] for i in range(num_methrows + annotation)],
+            print_grid=False,
+            subplot_titles=names,
+            vertical_spacing=0.1 if num_methrows < 10 else 0.01,
+            row_heights=[0.9 / num_methrows] * num_methrows + [0.1] * annotation,
+        )
+    else:
+        return plotly.subplots.make_subplots(
+            rows=num_methrows + annotation,
+            cols=1,
+            shared_xaxes=True,
+            specs=[[{"rowspan": num_methrows}], [None], [None], [None]] + [[{}]] * annotation,
+            print_grid=False,
+            vertical_spacing=0.1 if num_methrows < 10 else 0.01,
+            row_heights=[0.9, 0, 0, 0] + [0.1] * annotation,
+        )
+
+
+def create_browser_output(fig, outfile, window):
+    if outfile is None:
+        outfile = f"methylation_browser_{window.string}.html"
+    else:
+        from pathlib import Path
+
+        outfile = outfile.format(region=window.string)
+        p = Path(outfile)
+        Path.mkdir(p.parent, exist_ok=True, parents=True)
+
+    if outfile.endswith(".html"):
+        write_html_output(fig, outfile)
+    else:
+        try:
+            fig.write_image(outfile)
+        except ValueError as e:
+            sys.stderr.write("\n\nERROR: creating the image in this file format failed.\n")
+            sys.stderr.write("ERROR: creating in default html format instead.\n")
+            sys.stderr.write("ERROR: additional packages required. Detailed error:\n")
+            sys.stderr.write(str(e))
+            write_html_output(fig, outfile)
+
+
+def write_html_output(fig, outfile):
+    with open(outfile, "w+") as output:
+        output.write(
+            plotly.offline.plot(fig, output_type="div", show_link=False, include_plotlyjs="cdn")
+        )

methplotlib/version.py

@@ -0,0 +1 @@
+__version__ = "0.23.0"

methplotlib-0.23.0.data/scripts/allele_specific_modification

@@ -0,0 +1,42 @@
+#!python
+from argparse import ArgumentParser
+import pyranges as pr
+from methplotlib.helpers import methylation_pyranges_from_csv
+from methplotlib.differential.differential import main
+
+
+def get_args():
+    parser = ArgumentParser(
+        description="Check for modification differences using fisher exact test.")
+    parser.add_argument(
+        "-b", "--bed", help="Bed file to aggregate modifications on.", required=True)
+    parser.add_argument("methylation", nargs=2, help="Haplotype specific frequency files.")
+    parser.add_argument("-c", "--cutoff", help="FDR cutoff. Default: 0.05",
+                        default=0.05, type=float)
+    parser.add_argument(
+        "-o", "--out", help="File to write results to. Default: stdout.", default="-")
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = get_args()
+
+    gr = methylation_pyranges_from_csv(args.methylation[0])
+    gr2 = methylation_pyranges_from_csv(args.methylation[1])
+
+    df = pr.read_bed(args.bed, as_df=True)
+    bed = pr.PyRanges(df, int64=True).merge()
+    result = main(gr, gr2, bed)
+
+    result = result[result.FDR <= args.cutoff]
+
+    if args.out == "-":
+        import sys
+        result.to_csv(sys.stdout, sep="\t")  # index=False)
+    else:
+        from pathlib import Path
+
+        p = Path(args.out)
+        Path.mkdir(p.parent, exist_ok=True, parents=True)
+
+        result.to_csv(str(p), sep="\t")

methplotlib-0.23.0.data/scripts/differential_modification

@@ -0,0 +1,43 @@
+#!python
+from argparse import ArgumentParser
+import pyranges as pr
+from methplotlib.differential.differential import main
+from methplotlib.helpers import methylation_pyranges_from_csv
+
+
+def get_args():
+    parser = ArgumentParser(
+        description="Check for modification differences using fisher exact test.")
+    parser.add_argument(
+        "-b", "--bed", help="Bed file to aggregate modifications on.", required=True)
+    parser.add_argument("-A", "--Agroup", nargs='+', help="Frequencies of group A.")
+    parser.add_argument("-B", "--Bgroup", nargs='+', help="Frequencies of group B.")
+    parser.add_argument("-c", "--cutoff", help="FDR cutoff. Default: 0.05",
+                        default=0.05, type=float)
+    parser.add_argument(
+        "-o", "--out", help="File to write results to. Default: stdout.", default="-")
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+
+    args = get_args()
+
+    a_range = pr.concat([methylation_pyranges_from_csv(f) for f in args.Agroup])
+    b_range = pr.concat([methylation_pyranges_from_csv(f) for f in args.Bgroup])
+    bed = pr.read_bed(args.bed).merge()
+
+    gr = main(a_range, b_range, bed)
+
+    gr = gr[gr.ORFDR <= args.cutoff]
+
+    if args.out == "-":
+        import sys
+        gr.to_csv(sys.stdout, sep="\t")  # index=False)
+    else:
+        from pathlib import Path
+
+        p = Path(args.out)
+        Path.mkdir(p.parent, exist_ok=True, parents=True)
+
+        gr.to_csv(str(p), sep="\t")