methplotlib 0.21.2__tar.gz → 0.23.0__tar.gz

{methplotlib-0.21.2/methplotlib.egg-info → methplotlib-0.23.0}/PKG-INFO

@@ -1,23 +1,46 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: methplotlib
-Version: 0.21.2
-Summary: Plot methylation data obtained from nanopolish
+Version: 0.23.0
+Summary: Plot nucleotide modification data from nanopore sequencing
 Home-page: https://github.com/wdecoster/methplotlib
 Author: Wouter De Coster
 Author-email: decosterwouter@gmail.com
 License: MIT
-Keywords: nanopore
+Keywords: nanopore methylation modification
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.3
-Classifier: Programming Language :: Python :: 3.4
-Classifier: Programming Language :: Python :: 3.5
-Requires-Python: >=3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: plotly>=4.9.0
+Requires-Dist: numpy>=1.16.5
+Requires-Dist: pandas>=0.23.4
+Requires-Dist: pyranges>=0.0.77
+Requires-Dist: fisher>=0.1.9
+Requires-Dist: scikit-learn
+Requires-Dist: pyfaidx
+Requires-Dist: biopython
+Requires-Dist: pysam
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 [![Anaconda-Server Badge](https://anaconda.org/bioconda/methplotlib/badges/version.svg)](https://anaconda.org/bioconda/methplotlib)
 [![Anaconda-Server Badge](https://anaconda.org/bioconda/methplotlib/badges/license.svg)](https://anaconda.org/bioconda/methplotlib)
@@ -25,10 +48,11 @@ License-File: LICENSE
 # METHPLOTLIB
 
 This script generates a browser view on a window using data from  
-i) [nanopolish](https://github.com/jts/nanopolish), either as methylation calls or methylation frequencies (as processed by calculate_methylation_frequency.py). The methylation calls can additionally be phased using scripts/annotate_calls_by_phase.and scripts/split_calls_by_phase.py  
+i) [nanopolish](https://github.com/jts/nanopolish), either as methylation calls or methylation frequencies (as processed by calculate_methylation_frequency.py). The methylation calls can additionally be phased using scripts/annotate_calls_by_phase.py and scripts/split_calls_by_phase.py  
 ii) [nanocompore](https://github.com/tleonardi/nanocompore)  
 iii) in ont-cram format with MM/ML tags according to the SAM specifications  
-iv) in bedgraph format  
+iv) in bedgraph format, with 4 columns (chromosome, start, end, value) or the 5 columns written by `modkit pileup --bedgraph`  
+v) in bedMethyl format, as written by [modkit](https://github.com/nanoporetech/modkit) `pileup` (18 columns, with or without `--only-tabs`) or by `modbam2bed --extended` (14 columns). Files containing multiple modifications (e.g. 5mC and 5hmC) do not have to be split up: each modification gets its own track, and `--mods` can be used to restrict to some of them  
 
 ## INSTALLATION
 

{methplotlib-0.21.2 → methplotlib-0.23.0}/README.md

@@ -4,10 +4,11 @@
 # METHPLOTLIB
 
 This script generates a browser view on a window using data from  
-i) [nanopolish](https://github.com/jts/nanopolish), either as methylation calls or methylation frequencies (as processed by calculate_methylation_frequency.py). The methylation calls can additionally be phased using scripts/annotate_calls_by_phase.and scripts/split_calls_by_phase.py  
+i) [nanopolish](https://github.com/jts/nanopolish), either as methylation calls or methylation frequencies (as processed by calculate_methylation_frequency.py). The methylation calls can additionally be phased using scripts/annotate_calls_by_phase.py and scripts/split_calls_by_phase.py  
 ii) [nanocompore](https://github.com/tleonardi/nanocompore)  
 iii) in ont-cram format with MM/ML tags according to the SAM specifications  
-iv) in bedgraph format  
+iv) in bedgraph format, with 4 columns (chromosome, start, end, value) or the 5 columns written by `modkit pileup --bedgraph`  
+v) in bedMethyl format, as written by [modkit](https://github.com/nanoporetech/modkit) `pileup` (18 columns, with or without `--only-tabs`) or by `modbam2bed --extended` (14 columns). Files containing multiple modifications (e.g. 5mC and 5hmC) do not have to be split up: each modification gets its own track, and `--mods` can be used to restrict to some of them  
 
 ## INSTALLATION
 

{methplotlib-0.21.2 → methplotlib-0.23.0}/methplotlib/annotation.py

@@ -31,11 +31,14 @@ def good_record(line, chromosome):
     """
     Filtering on the gtf lines
     by checking for right chromosome and right feature type
+
+    The chromosome has to match the entire field, as a prefix match would
+    e.g. also accept chr10 up to chr19 and chr1_KI270706v1_random for chr1
     """
-    if line.startswith(str(chromosome)) and line.split("\t")[2] in ["exon", "gene"]:
-        return True
-    else:
+    if line.startswith("#"):
         return False
+    fields = line.split("\t")
+    return len(fields) > 2 and fields[0] == str(chromosome) and fields[2] in ["exon", "gene"]
 
 
 def get_features(gtfline, type="gtf"):
@@ -54,8 +57,10 @@ def parse_attributes(attributes, type="gtf"):
     """
     attribute_delimiter = {"gtf": "; ", "gff": ";"}
     kv_delimiter = {"gtf": " ", "gff": "="}
+    # the last attribute of a record keeps the trailing delimiter,
+    # which would otherwise end up in the name shown when hovering
     info = {
-        i.split(kv_delimiter[type])[0]: i.split(kv_delimiter[type])[1].replace('"', "")
+        i.split(kv_delimiter[type])[0]: i.split(kv_delimiter[type])[1].replace('"', "").rstrip(";")
         for i in attributes.split(attribute_delimiter[type])
         if i.startswith(("gene_name", "transcript_id", "locus_tag"))
     }
@@ -67,17 +72,19 @@ def parse_attributes(attributes, type="gtf"):
 
 def transcripts_in_window(df, window, feature="transcript"):
     """
-    Return the transcript names for which
-    either the end or the begin of an exon is within the window
+    Return the transcript names of the exons overlapping the window
+
+    Testing for overlap rather than for a begin or end within the window,
+    as otherwise a feature spanning the entire window would be missed
     """
     return df.loc[
-        df["begin"].between(window.begin, window.end) | df["end"].between(window.begin, window.end),
+        (df["begin"] <= window.end) & (df["end"] >= window.begin),
         feature,
     ].unique()
 
 
 def assign_colors_to_genes(transcripts):
-    genes = set([t.gene for t in transcripts])
+    genes = dict.fromkeys([t.gene for t in transcripts])
     colordict = {g: c for g, c in zip(genes, plcolors * 100)}
     for t in transcripts:
         t.color = colordict[t.gene]
@@ -90,12 +97,24 @@ def parse_annotation(gtff, window, simplify=False):
     """
     type = annot_file_sniffer(gtff)
     logging.info(f"Parsing {type} file...")
+    chromosomes_seen = set()
+    records = []
+    for line in open_gtf(gtff):
+        if good_record(line, window.chromosome):
+            records.append(get_features(line, type=type))
+        elif line.strip() and not line.startswith("#"):
+            chromosomes_seen.add(line.split("\t")[0])
+    if not records and chromosomes_seen:
+        sys.stderr.write(
+            f"\nWARNING: no exons or genes of chromosome '{window.chromosome}' "
+            f"in {gtff}.\n"
+            f"The annotation contains {len(chromosomes_seen)} chromosomes, "
+            f"such as {', '.join(sorted(chromosomes_seen)[:5])}.\n"
+            "Do the chromosome names in --window match those in your annotation "
+            "(e.g. '15' versus 'chr15')?\n"
+        )
     df = pd.DataFrame(
-        data=[
-            get_features(line, type=type)
-            for line in open_gtf(gtff)
-            if good_record(line, window.chromosome)
-        ],
+        data=records,
         columns=["chromosome", "begin", "end", "strand", "gene", "transcript"],
     )
     logging.info("Loaded GTF file, processing...")

{methplotlib-0.21.2 → methplotlib-0.23.0}/methplotlib/import_methylation.py

@@ -3,7 +3,7 @@ import pyranges as pr
 import numpy as np
 import sys
 import logging
-from methplotlib.utils import file_sniffer, flatten
+from methplotlib.utils import file_sniffer, flatten, bedmethyl_separator
 from itertools import repeat
 
 
@@ -16,7 +16,7 @@ class Modification(object):
         self.start_end_table = start_end_table
 
 
-def get_data(methylation_files, names, window, smoothen=5):
+def get_data(args, window):
     """
     Import methylation data from all files in the list methylation_files
 
@@ -31,10 +31,10 @@ def get_data(methylation_files, names, window, smoothen=5):
     data is extracted within the window args.window
     Frequencies are smoothened using a sliding window
     """
-    return flatten([read_mods(f, n, window, smoothen) for f, n in zip(methylation_files, names)])
+    return flatten([read_mods(f, n, window, args) for f, n in zip(args.methylation, args.names)])
 
 
-def read_mods(filename, name, window, smoothen=5):
+def read_mods(filename, name, window, args):
     """
     converts a file from nanopolish to a pandas dataframe
     input can be from calculate_methylation_frequency
@@ -48,17 +48,17 @@ def read_mods(filename, name, window, smoothen=5):
     logging.info(f"File {filename} is of type {file_type}")
     try:
         if file_type.startswith("nanopolish"):
-            return parse_nanopolish(filename, file_type, name, window, smoothen=smoothen)
+            return parse_nanopolish(filename, file_type, name, window, smoothen=args.smooth)
         elif file_type == "nanocompore":
             return [parse_nanocompore(filename, name, window)]
         elif file_type in ["cram", "bam"]:
-            return parse_cram(filename, file_type, name, window)
+            return parse_cram(filename, file_type, name, window, args.mods)
         elif file_type == "bedgraph":
-            return [parse_bedgraph(filename, name, window)]
+            return parse_bedgraph(filename, name, window)
         elif file_type == "bedmethyl_extended":
-            return [parse_bedmethyl(filename, name, window, smoothen=smoothen, flavor="modbam2bed")]
+            return parse_bedmethyl(filename, name, window, smoothen=args.smooth, flavor="modbam2bed", mods_of_interest=args.mods)
         elif file_type == "bedmethyl":
-            return [parse_bedmethyl(filename, name, window, smoothen=smoothen, flavor="modkit")]
+            return parse_bedmethyl(filename, name, window, smoothen=args.smooth, flavor="modkit", mods_of_interest=args.mods)
     except Exception as e:
         logging.error(f"Error processing {filename}.")
         logging.error(e, exc_info=True)
@@ -98,7 +98,9 @@ def parse_nanopolish(filename, file_type, name, window, smoothen=5):
                 sys.stderr.write("Please index with 'tabix -S1 -s1 -b3 -e4'.\n")
             else:
                 sys.stderr.write("Please index with 'tabix -S1 -s1 -b2 -e3'.\n")
-            iter_csv = pd.read_csv(filename, sep="\t", iterator=True, chunksize=1e6)
+            iter_csv = pd.read_csv(
+                filename, sep="\t", iterator=True, chunksize=int(1e6), dtype={"chromosome": str}
+            )
             table = pd.concat(
                 [chunk[chunk["chromosome"] == window.chromosome] for chunk in iter_csv]
             )
@@ -166,6 +168,9 @@ def parse_nanopolish(filename, file_type, name, window, smoothen=5):
         called_sites = table.called_sites
         chromosome = table.Chromosome.values[0]
 
+        # errors are ignored as a reformatted file may not have all of these,
+        # and numeric_only skips any column that was added to the file,
+        # which would make the mean below fail
         table = table.drop(
             columns=[
                 "Chromosome",
@@ -175,13 +180,14 @@ def parse_nanopolish(filename, file_type, name, window, smoothen=5):
                 "called_sites",
                 "called_sites_methylated",
                 "group_sequence",
-            ]
+            ],
+            errors="ignore",
         )
         return [
             Modification(
                 table=table.sort_values("pos")
                 .groupby("pos")
-                .mean()
+                .mean(numeric_only=True)
                 .rolling(window=smoothen, center=True)
                 .mean()
                 .assign(Chromosome=chromosome),
@@ -204,9 +210,13 @@ def parse_nanocompore(filename, name, window):
     table = pd.read_csv(filename, sep="\t", usecols=nanocompore_columns_of_interest)
     if window:
         table = table[table["ref_id"] == window.chromosome]
+    # a record at the end of the window extends the line to the edge of the plot,
+    # its pvalue of 1.0 is filled in below
     return Modification(
-        table=table.sort_values("pos")
-        .append({"pos": window.end}, ignore_index=True)
+        table=pd.concat(
+            [table.sort_values("pos"), pd.DataFrame({"pos": [window.end]})],
+            ignore_index=True,
+        )
         .drop(columns="ref_id")
         .fillna(1.0),
         data_type="nanocompore",
@@ -215,7 +225,56 @@ def parse_nanocompore(filename, name, window):
     )
 
 
+def no_records_warning(filename, window):
+    """
+    Warn that a file has no data in the window.
+    This is not fatal, as other files may still have data here.
+    """
+    message = f"No records for {filename} in {window.chromosome}:{window.begin}-{window.end}!"
+    logging.warning(message)
+    sys.stderr.write(f"\nWARNING: {message}\nThis dataset is skipped.\n")
+
+
+def read_in_chunks(filename, window, sep, usecols, index_suggestion):
+    """
+    Read a headerless file in chunks, keeping only the lines of the chromosome of interest.
+
+    This is the slow path, used when no tabix index is available.
+    The chromosome is read as a string, as that is how the window stores it,
+    and to avoid mixed types when a file has both numeric and non-numeric chromosomes.
+    """
+    logging.info(f"Reading {filename} slowly by splitting the file in chunks.")
+    sys.stderr.write(f"\nReading {filename} would be faster with bgzip and '{index_suggestion}'.\n")
+    iter_csv = pd.read_csv(
+        filename,
+        sep=sep,
+        iterator=True,
+        chunksize=int(1e6),
+        header=None,
+        usecols=usecols,
+        dtype={0: str},
+    )
+    chromosomes_seen = set()
+    chunks = []
+    for chunk in iter_csv:
+        chromosomes_seen.update(chunk[0].unique())
+        chunks.append(chunk[chunk[0] == window.chromosome])
+    if window.chromosome not in chromosomes_seen:
+        sys.stderr.write(
+            f"\nWARNING: chromosome '{window.chromosome}' not found in {filename}.\n"
+            f"The file contains {len(chromosomes_seen)} chromosomes, "
+            f"such as {', '.join(sorted(chromosomes_seen)[:5])}.\n"
+            "Do the chromosome names in --window match those in your file "
+            "(e.g. '1' versus 'chr1')?\n"
+        )
+    return pd.concat(chunks)
+
+
 def parse_bedgraph(filename, name, window):
+    # a bedgraph from `modkit pileup --bedgraph` has a fifth column with the coverage,
+    # which is ignored here
+    colnames = ["Chromosome", "Start", "End", "Value"]
+    usecols = [0, 1, 2, 3]
     if window:
         from pathlib import Path
 
@@ -240,46 +299,45 @@ def parse_bedgraph(filename, name, window):
                 tabix_stream.stdout,
                 sep="\t",
                 header=None,
-                names=["Chromosome", "Start", "End", "Value"],
+                usecols=usecols,
+                names=colnames,
+                dtype={0: str},
             )
         else:
-            logging.info(f"Reading {filename} slowly by splitting the file in chunks.")
-            sys.stderr.write(
-                f"\nReading {filename} would be faster with bgzip and 'tabix -p bed'.\n"
-            )
-            iter_csv = pd.read_csv(
+            table = read_in_chunks(
                 filename,
+                window,
                 sep="\t",
-                iterator=True,
-                chunksize=1e6,
-                header=None,
-                names=["Chromosome", "Start", "End", "Value"],
-            )
-            table = pd.concat(
-                [chunk[chunk["Chromosome"] == window.chromosome] for chunk in iter_csv]
-            )
+                usecols=usecols,
+                index_suggestion="tabix -p bed",
+            ).rename(columns=dict(zip(usecols, colnames)))
     else:
         table = pd.read_csv(
             filename,
             sep="\t",
             header=None,
-            names=["Chromosome", "Start", "End", "Value"],
+            usecols=usecols,
+            names=colnames,
+            dtype={0: str},
         )
     gr = pr.PyRanges(table)
     logging.info("Read the file in a dataframe.")
     if window:
         gr = gr[window.chromosome, window.begin : window.end]
         if len(gr.df) == 0:
-            sys.exit(f"No records for {filename} in {window.string}!\n")
-    return Modification(
-        table=gr.df.sort_values("Start"),
-        data_type="bedgraph",
-        name=name,
-        called_sites=len(table),
-    )
+            no_records_warning(filename, window)
+            return []
+    return [
+        Modification(
+            table=gr.df.sort_values("Start"),
+            data_type="bedgraph",
+            name=name,
+            called_sites=len(table),
+        )
+    ]
 
 
-def parse_bedmethyl(filename, name, window, smoothen=5, flavor="modkit"):
+def parse_bedmethyl(filename, name, window, smoothen=5, flavor="modkit", mods_of_interest=None):
     if not flavor in ["modkit", "modbam2bed"]:
         sys.exit(f"ERROR: flavor {flavor} not supported for bedmethyl.")
     if flavor == "modkit":
@@ -287,8 +345,7 @@ def parse_bedmethyl(filename, name, window, smoothen=5, flavor="modkit"):
             0: "Chromosome",
             1: "Start",
             2: "End",
-            # if multiple types of modification should be supported additionally column 3 has to be read
-            # 3: "Modification",
+            3: "Modification",
             10: "Frequency",
         }
     else:
@@ -296,12 +353,13 @@ def parse_bedmethyl(filename, name, window, smoothen=5, flavor="modkit"):
             0: "Chromosome",
             1: "Start",
             2: "End",
-            # if multiple types of modification should be supported additionally column 3 has to be read
-            # 3: "Modification",
+            3: "Modification",
             11: "canonical",
             12: "modified",
         }
-    usecols = colnames.keys()
+    usecols = list(colnames.keys())
+    # modkit separates the columns after BED9 by spaces, unless run with --only-tabs
+    sep = bedmethyl_separator(filename)
     if window:
         from pathlib import Path
 
@@ -324,39 +382,34 @@ def parse_bedmethyl(filename, name, window, smoothen=5, flavor="modkit"):
                 raise
             table = pd.read_csv(
                 tabix_stream.stdout,
-                sep="\t",
+                sep=sep,
                 header=None,
                 usecols=usecols,
+                dtype={0: str},
             ).rename(columns=colnames)
         else:
-            logging.info(f"Reading {filename} slowly by splitting the file in chunks.")
-            sys.stderr.write(
-                f"\nReading {filename} would be faster with bgzip and 'tabix -p bed'.\n"
-            )
-            iter_csv = pd.read_csv(
+            table = read_in_chunks(
                 filename,
-                sep="\t",
-                iterator=True,
-                chunksize=1e6,
-                header=None,
+                window,
+                sep=sep,
                 usecols=usecols,
-            )
-            table = pd.concat([chunk[chunk[0] == window.chromosome] for chunk in iter_csv]).rename(
-                columns=colnames
-            )
+                index_suggestion="tabix -p bed",
+            ).rename(columns=colnames)
     else:
         table = pd.read_csv(
             filename,
-            sep="\t",
+            sep=sep,
             header=None,
             usecols=usecols,
+            dtype={0: str},
         ).rename(columns=colnames)
     gr = pr.PyRanges(table)
     logging.info("Read the file in a dataframe.")
     if window:
         gr = gr[window.chromosome, window.begin : window.end]
         if len(gr.df) == 0:
-            sys.exit(f"No records for {filename} in {window.string}!\n")
+            no_records_warning(filename, window)
+            return []
     table = gr.df.sort_values("Start")
     if flavor == "modkit":
         table["modified_frequency"] = table["Frequency"] / 100
@@ -366,15 +419,54 @@ def parse_bedmethyl(filename, name, window, smoothen=5, flavor="modkit"):
     table.drop(
         columns=["Chromosome", "End", "canonical", "modified"], inplace=True, errors="ignore"
     )
-    return Modification(
-        table=table.rolling(window=smoothen, center=True).mean(),
-        data_type="bedmethyl_extended",
-        name=name,
-        called_sites=len(table),
+    if mods_of_interest:
+        # mods of interest is in the form of C+m or C+h (as in the cram file)
+        # however, the bedmethyl file has the modifications in the form of m or h
+        # so we need to split the mods in mods_of_interest on the '+' and select the second part
+        mods = [m.split("+")[1] if "+" in m else m for m in mods_of_interest.split(",")]
+        modifications_found = sorted(table["Modification"].unique())
+        table = table[table["Modification"].isin(mods)]
+        if len(table) == 0:
+            sys.exit(
+                f"\n\nERROR: No records left for {filename} after selecting --mods!\n"
+                f"Modifications in this file: {', '.join(modifications_found)}\n"
+            )
+
+    return [
+        Modification(
+            table=smoothen_frequency(sub_df.drop(columns=["Modification"]), smoothen),
+            data_type="bedmethyl_extended",
+            name=f"{name}_{mod}",
+            called_sites=len(sub_df),
+        )
+        for mod, sub_df in table.groupby("Modification") if len(sub_df) > 0
+    ]
+
+
+def smoothen_frequency(table, smoothen):
+    """
+    Rolling average of the modification frequency
+
+    Only the frequency is averaged, as averaging the positions as well would
+    plot the records in between the sites they were observed at.
+    Averaging over fewer records at the edges avoids losing those records.
+    """
+    table["modified_frequency"] = (
+        table["modified_frequency"].rolling(window=smoothen, center=True, min_periods=1).mean()
     )
+    return table
+
 
+def parse_cram(filename, filetype, name, window, mods_of_interest=None):
+    """
+    Extracts modified positions from a CRAM file
 
-def parse_cram(filename, filetype, name, window):
+    :param filename: str, path to the CRAM file
+    :param filetype: str, either 'cram' or 'bam'
+    :param name: str, name for the trace/sample
+    :param window: Region object to extract data for from the file
+    :param mods_of_interest: list of str, optional, list of modifications to extract
+    """
     import pysam
 
     mode = "rc" if filetype == "cram" else "rb"
@@ -393,6 +485,15 @@ def parse_cram(filename, filetype, name, window):
         .sort_values(["read_name", "pos"])
     )
 
+
+    if mods_of_interest:
+        mods_found = sorted(df["mod"].astype(str).unique())
+        df = df[df["mod"].isin(mods_of_interest.split(","))]
+        if len(df) == 0:
+            sys.exit(
+                f"\n\nERROR: No records left for {filename} after selecting --mods!\n"
+                f"Modifications in this file: {', '.join(mods_found)}\n"
+            )
     return [
         Modification(
             table=sub_df,
@@ -403,7 +504,7 @@ def parse_cram(filename, filetype, name, window):
                 start_stops, columns=["read_name", "posmin", "posmax"]
             ).set_index("read_name"),
         )
-        for mod, sub_df in df.groupby("mod")
+        for mod, sub_df in df.groupby("mod") if len(sub_df) > 0
     ]
 
 
@@ -415,7 +516,7 @@ def get_modified_reference_positions(read):
         offset = 0
         for context in read.get_tag(modtag).split(";"):
             if context:
-                basemod = context.split(",", 1)[0]
+                basemod = context.split(",", 1)[0].replace('?', '')
                 if "-" in basemod:
                     sys.exit(
                         "ERROR: modifications on negative strand currently unsupported.\n"