clipnet 0.2.1__py3-none-any.whl

clipnet/__init__.py

@@ -0,0 +1,9 @@
+# __init__.py
+# Adam He <adamyhe@gmail.com>
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("clipnet")
+except PackageNotFoundError:
+    __version__ = "unknown"

clipnet/_calculate_activator_interaction.py

@@ -0,0 +1,159 @@
+## NOT IMPLEMENTED
+
+"""
+Calculate contribution scores using shap.DeepExplainer.
+"""
+
+import argparse
+import gc
+
+import numpy as np
+import shap
+import tensorflow as tf
+
+import .utils
+
+# This will fix an error message for running tf.__version__==2.5
+shap.explainers._deep.deep_tf.op_handlers["AddV2"] = (
+    shap.explainers._deep.deep_tf.passthrough
+)
+tf.compat.v1.disable_v2_behavior()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("core_promoter_seq", type=str, help="")
+    parser.add_argument("motif", type=str, help="Where to write DeepSHAP scores.")
+    parser.add_argument(
+        "--model_dir",
+        type=str,
+        default="ensemble_models",
+        help="Directory to load models from",
+    )
+    parser.add_argument(
+        "--mode",
+        type=str,
+        default="quantity",
+        help="Calculate interaction strength using quantity or DFIM",
+    )
+    parser.add_argument("--gpu", action="store_true", help="Enable GPU.")
+    parser.add_argument(
+        "--use_specific_gpu",
+        type=int,
+        default=0,
+        help="Index of GPU to use (starting from 0). Does nothing if --gpu is not set.",
+    )
+    parser.add_argument(
+        "--n_subset",
+        type=int,
+        default=100,
+        help="Maximum number of sequences to use as background. \
+            Default is 100 to ensure reasonably fast compute on large datasets.",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=617,
+        help="Random seed for selecting background sequences.",
+    )
+    args = parser.parse_args()
+    np.random.seed(args.seed)
+
+    assert args.mode in [
+        "quantity",
+        "dfim",
+    ], "mode must be either quantity or dfim."
+
+    # Load sequences ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    concat_ref = "".join(shuffled_ref)
+    embedded_promoter = (
+        ushuffle.shuffle(
+            concat_ref, int(np.floor((1000 - len(core_promoter_seq)) / 2)), 2
+        )
+        + core_promoter_seq
+        + ushuffle.shuffle(
+            concat_ref, int(np.ceil((1000 - len(core_promoter_seq)) / 2)), 2
+        )
+    )
+    seqs_to_explain = np.array([utils.OneHotDNA(seq).onehot for seq in sequences])
+
+    # Perform dinucleotide shuffle on n_subset random sequences
+    if len(sequences) < args.n_subset:
+        args.n_subset = len(sequences)
+    reference = [
+        sequences[i]
+        for i in np.random.choice(
+            np.array(range(len(sequences))), size=args.n_subset, replace=False
+        )
+    ]
+    shuffled_reference = [
+        utils.kshuffle(rec.seq, random_seed=args.seed)[0] for rec in reference
+    ]
+
+    # One-hot encode shuffled sequences
+    onehot_reference = np.array(
+        [utils.OneHotDNA(seq).onehot for seq in shuffled_reference]
+    )
+
+    # Load model and create explainer ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    if args.gpu:
+        gpus = tf.config.experimental.list_physical_devices("GPU")
+        for gpu in gpus:
+            tf.config.experimental.set_memory_growth(gpu, True)
+        tf.config.set_visible_devices(gpus[args.use_specific_gpu], "GPU")
+    else:
+        tf.config.set_visible_devices([], "GPU")
+
+    if args.model_fp is None:
+        models = [
+            tf.keras.models.load_model(f"{args.model_dir}/fold_{i}.h5", compile=False)
+            for i in range(1, 10)
+        ]
+    else:
+        models = [tf.keras.models.load_model(args.model_fp, compile=False)]
+    if args.mode == "quantity":
+        contrib = [model.output[1] for model in models]
+    else:
+        contrib = [
+            tf.reduce_mean(
+                tf.stop_gradient(tf.nn.softmax(model.output[0], axis=-1))
+                * model.output[0],
+                axis=-1,
+                keepdims=True,
+            )
+            for model in models
+        ]
+    explainers = [
+        shap.DeepExplainer((model.input, contrib), onehot_reference)
+        for (model, contrib) in zip(models, contrib)
+    ]
+
+    # Calculate scores ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    raw_explanations = {i: [] for i in range(len(explainers))}
+    batch_size = 256
+    for i in range(len(explainers)):
+        for j in range(0, len(seqs_to_explain), batch_size):
+            shap_values = explainers[i].shap_values(seqs_to_explain[j : j + batch_size])
+            raw_explanations[i].append(shap_values)
+            gc.collect()
+
+    concat_explanations = []
+    for k in raw_explanations.keys():
+        concat_explanations.append(
+            np.concatenate([exp for exp in raw_explanations[k]], axis=1).sum(axis=0)
+        )
+
+    mean_explanations = np.array(concat_explanations).mean(axis=0)
+    scaled_explanations = mean_explanations * seqs_to_explain
+
+    # Save scores ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    np.savez_compressed(args.score_fp, scaled_explanations.swapaxes(1, 2))
+    np.savez_compressed(args.seq_fp, (seqs_to_explain / 2).astype(int).swapaxes(1, 2))
+
+
+if __name__ == "__main__":
+    main()

clipnet/_calculate_dataset_params.py

@@ -0,0 +1,123 @@
+## NOT IMPLEMENTED. PART OF THE FITTING SCRIPTS
+
+"""
+Calculates dataset parameters needed by clipnet. Supply a path to the processed data
+and an output directory. This script will write json files to the output directory
+for use in model training.
+"""
+
+import argparse
+import json
+import os
+
+import numpy as np
+
+
+def write_dataset_params(i, datadir, outdir):
+    outdir = f"{outdir}/f{i}/"
+    os.makedirs(outdir, exist_ok=True)
+
+    test_folds = [i]
+    val_folds = [i % 9 + 1]
+    train_folds = [j for j in range(1, 10) if j not in test_folds + val_folds]
+    print(train_folds, val_folds, test_folds)
+
+    dataset_params = {
+        "train_seq": [
+            os.path.join(datadir, f"concat_sequence_{fold}.npz") for fold in train_folds
+        ],
+        "train_procap": [
+            os.path.join(datadir, f"concat_procap_{fold}.npz") for fold in train_folds
+        ],
+        "val_seq": [
+            os.path.join(datadir, f"concat_sequence_{fold}.npz") for fold in val_folds
+        ],
+        "val_procap": [
+            os.path.join(datadir, f"concat_procap_{fold}.npz") for fold in val_folds
+        ],
+        "test_seq": [
+            os.path.join(datadir, f"concat_sequence_{fold}.npz") for fold in test_folds
+        ],
+        "test_procap": [
+            os.path.join(datadir, f"concat_procap_{fold}.npz") for fold in test_folds
+        ],
+    }
+
+    dataset_params["n_train_folds"] = len(train_folds)
+    dataset_params["n_val_folds"] = len(val_folds)
+    dataset_params["n_test_folds"] = len(test_folds)
+
+    # Calculate n_samples_per_chunk
+    dataset_params["n_samples_per_train_fold"] = [
+        np.load(f)["arr_0"].shape[0] for f in dataset_params["train_procap"]
+    ]
+    dataset_params["n_samples_per_val_fold"] = [
+        np.load(f)["arr_0"].shape[0] for f in dataset_params["val_procap"]
+    ]
+    dataset_params["n_samples_per_test_fold"] = [
+        np.load(f)["arr_0"].shape[0] for f in dataset_params["test_procap"]
+    ]
+
+    dataset_params["window_length"] = np.load(dataset_params["train_seq"][0])["arr_0"][
+        0
+    ].shape[0]
+
+    dataset_params["pad"] = int(dataset_params["window_length"] / 4)
+    dataset_params["output_length"] = int(
+        2 * (dataset_params["window_length"] - 2 * dataset_params["pad"])
+    )
+
+    dataset_params["weight"] = 1 / 500
+
+    output_fp = os.path.join(outdir, "dataset_params.json")
+
+    with open(output_fp, "w") as handle:
+        json.dump(dataset_params, handle, indent=4, sort_keys=True)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("datadir", type=str, help="directory containing data")
+    parser.add_argument(
+        "outdir",
+        type=str,
+        help="directory to save dataset params to (where models will be saved)",
+    )
+    parser.add_argument(
+        "--threads",
+        type=int,
+        default=9,
+        help="number of threads to use. Only used if not --fold is not selected.",
+    )
+    parser.add_argument(
+        "--fold",
+        type=int,
+        default=None,
+        help="fold to calculate dataset params for (will only run one).",
+    )
+    args = parser.parse_args()
+    if args.threads <= 0:
+        raise ValueError("--threads must be a positive integer")
+    if args.fold is not None:
+        write_dataset_params(args.fold, args.datadir, args.outdir)
+    else:
+        if args.threads == 1:
+            for i in range(1, 10):
+                write_dataset_params(i, args.datadir, args.outdir)
+        elif args.threads > 1:
+            import itertools
+            import multiprocessing as mp
+
+            with mp.Pool(9) as p:
+                p.starmap(
+                    write_dataset_params,
+                    zip(
+                        range(1, 10),
+                        itertools.repeat(args.datadir),
+                        itertools.repeat(args.outdir),
+                    ),
+                )
+
+
+if __name__ == "__main__":
+    main()

clipnet/_calculate_ism_shuffle.py

@@ -0,0 +1,157 @@
+## NOT IMPLEMENTED
+
+"""
+Calculate ISM shuffle scores.
+"""
+
+import argparse
+import logging
+import os
+
+import numpy as np
+import pyfastx
+import tqdm
+from scipy.spatial.distance import cdist
+
+import utils
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "4"
+logging.getLogger("tensorflow").setLevel(logging.FATAL)
+import clipnet
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("fasta_fp", type=str, help="Fasta file path.")
+    parser.add_argument(
+        "score_fp", type=str, help="Where to write ISM shuffle results."
+    )
+    parser.add_argument(
+        "--model_dir",
+        type=str,
+        default="ensemble_models/",
+        help="directory to load models from.",
+    )
+    parser.add_argument(
+        "--gpu",
+        type=int,
+        default=None,
+        help="Index of GPU to use (starting from 0). If not invoked, uses CPU.",
+    )
+    parser.add_argument(
+        "--n_shuffles",
+        type=int,
+        default=5,
+        help="Number of shuffles/mutations to perform for each position.",
+    )
+    parser.add_argument(
+        "--mut_size", type=int, default=10, help="Size of mutations to use."
+    )
+    parser.add_argument(
+        "--edge_padding",
+        type=int,
+        default=50,
+        help="Number of positions from edge that we'll skip mutating on.",
+    )
+    parser.add_argument(
+        "--corr_pseudocount",
+        type=float,
+        default=1e-6,
+        help="Pseudocount for correlation calculation.",
+    )
+    parser.add_argument(
+        "--log_quantity_pseudocount",
+        type=float,
+        default=1e-3,
+        help="Pseudocount for log quantity calculation.",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=617,
+        help="Random seed for generating mutations.",
+    )
+    parser.add_argument(
+        "--silence",
+        action="store_true",
+        help="Disables progress bars and other non-essential print statements.",
+    )
+    args = parser.parse_args()
+    np.random.seed(args.seed)
+
+    # Load models ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    nn = (
+        clipnet.CLIPNET(n_gpus=1, use_specific_gpu=args.gpu)
+        if args.gpu is not None
+        else clipnet.CLIPNET(n_gpus=0)
+    )
+    ensemble = nn.construct_ensemble(args.model_dir, silence=args.silence)
+
+    # Load sequences ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    sequences = pyfastx.Fasta(args.fasta_fp)
+    seqs_twohot = utils.get_twohot_fasta_sequences(args.fasta_fp, silence=args.silence)
+
+    # Calculate ISM shuffle scores ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    wt_pred = ensemble.predict(seqs_twohot, batch_size=256, verbose=0)
+    corr_scores = []
+    log_quantity_scores = []
+    for i in tqdm.tqdm(
+        range(len(sequences)),
+        desc="Calculating ISM shuffle scores",
+        disable=args.silence,
+    ):
+        corr_score = []
+        log_quantity_score = []
+        rec = sequences[i]
+        positions = range(args.edge_padding, len(rec.seq) - args.edge_padding)
+        for shuffle in range(args.n_shuffles):
+            mutated_seqs = []
+            for pos in positions:
+                mut = utils.kshuffle(rec.seq, random_seed=args.seed)[0][: args.mut_size]
+                mutated_seq = (
+                    rec.seq[0 : pos - int(len(mut) / 2)]
+                    + mut
+                    + rec.seq[pos + int(len(mut) / 2) :]
+                )
+                mutated_seqs.append(mutated_seq)
+            mut_pred = ensemble.predict(
+                np.array([utils.TwoHotDNA(seq).twohot for seq in mutated_seqs]),
+                batch_size=256,
+                verbose=0,
+            )
+            mut_corr = (
+                1
+                - cdist(
+                    np.array(
+                        [wt_pred[0][i, :]] * len(positions)
+                        + np.random.normal(
+                            0, args.corr_pseudocount, mut_pred[0].shape[1]
+                        )
+                    ),
+                    mut_pred[0]
+                    + np.random.normal(0, args.corr_pseudocount, mut_pred[0].shape[1]),
+                    metric="correlation",
+                )[0, :]
+            )
+            mut_log_quantity = np.log10(
+                mut_pred[1] + args.log_quantity_pseudocount
+            ) - np.log10(wt_pred[1][i] + args.log_quantity_pseudocount)
+            corr_score.append(mut_corr)
+            log_quantity_score.append(mut_log_quantity)
+        corr_scores.append(np.array(corr_score).mean(axis=0))
+        log_quantity_scores.append(np.array(log_quantity_score).mean(axis=0))
+
+    # Save scores ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    np.savez_compressed(
+        args.score_fp,
+        corr_ism_shuffle=np.array(corr_scores),
+        log_quantity_ism_shuffle=np.array(log_quantity_scores),
+    )
+
+
+if __name__ == "__main__":
+    main()

clipnet/_calculate_performance_metrics.py

@@ -0,0 +1,150 @@
+## NOT IMPLEMENTED
+
+"""
+This script calculates performance metrics given a set of predictions and ground truth.
+"""
+
+import argparse
+
+import h5py
+import numpy as np
+import pandas as pd
+from scipy.spatial.distance import jensenshannon
+from scipy.stats import pearsonr, spearmanr
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "predictions",
+        type=str,
+        help="An hdf5 file containing predictions (track, quantity).",
+    )
+    parser.add_argument(
+        "observed",
+        type=str,
+        help="A csv(.gz), npy, or npz file containing the observed procap tracks.",
+    )
+    parser.add_argument(
+        "output",
+        type=str,
+        help="An hdf5 file to write the performance metrics to.",
+    )
+    args = parser.parse_args()
+
+    # Load predictions
+    with h5py.File(args.predictions, "r") as hf:
+        track = hf["track"][:]
+        if len(hf["quantity"].shape) == 2:
+            quantity = hf["quantity"][:, 0]
+        else:
+            quantity = hf["quantity"][:]
+
+    # Load observed data
+    if args.observed.endswith(".npz") or args.observed.endswith(".npy"):
+        observed = np.load(args.observed)
+        if args.observed.endswith(".npz"):
+            observed = observed["arr_0"]
+    elif args.observed.endswith(".csv.gz") or args.observed.endswith(".csv"):
+        observed = pd.read_csv(args.observed, header=None, index_col=0).to_numpy()
+    else:
+        raise ValueError(
+            f"File with observed PRO-cap data ({args.observed}) must be numpy or csv format."
+        )
+
+    # Validate dimensions
+    if track.shape[0] != observed.shape[0]:
+        raise ValueError(
+            f"n predictions ({track.shape[0]}) and n observed ({observed.shape[0]}) do not match."
+        )
+    if track.shape[1] > observed.shape[1]:
+        raise ValueError(
+            f"Predicted tracks ({track.shape[1]}) are longer than observed ({observed.shape[1]})."
+        )
+    if (observed.shape[1] - track.shape[1]) % 4 != 0:
+        raise ValueError(
+            f"Padding around predicted tracks ({observed.shape[1] - track.shape[1]}) must be divisible by 4."
+        )
+
+    # Trim off padding for observed tracks
+    start = (observed.shape[1] - track.shape[1]) // 4
+    end = observed.shape[1] // 2 - start
+    observed_clipped = observed[
+        :,
+        np.r_[start:end, observed.shape[1] // 2 + start : observed.shape[1] // 2 + end],
+    ]
+
+    # Benchmark directionality
+    track_directionality = np.log1p(
+        track[:, : track.shape[1] // 2].sum(axis=1)
+    ) - np.log1p(track[:, track.shape[1] // 2 :].sum(axis=1))
+    observed_directionality = np.log1p(
+        observed_clipped[:, : observed_clipped.shape[1] // 2].sum(axis=1)
+    ) - np.log1p(observed_clipped[:, observed_clipped.shape[1] // 2 :].sum(axis=1))
+    directionality_pearson = pearsonr(track_directionality, observed_directionality)
+
+    # Benchmark TSS position
+    strand_break = track.shape[1] // 2
+    pred_tss = np.concatenate(
+        [track[:, :strand_break].argmax(axis=1), track[:, strand_break:].argmax(axis=1)]
+    )
+    obs_tss = np.concatenate(
+        [
+            observed_clipped[:, :strand_break].argmax(axis=1),
+            observed_clipped[:, strand_break:].argmax(axis=1),
+        ]
+    )
+    tss_pos_pearson = pearsonr(pred_tss, obs_tss)
+
+    # Benchmark profile
+    track_pearson = pd.DataFrame(track).corrwith(pd.DataFrame(observed_clipped), axis=1)
+    track_js_distance = jensenshannon(track, observed_clipped, axis=1)
+
+    # Benchmark quantity
+    quantity_log_pearson = pearsonr(
+        np.log1p(quantity), np.log1p(observed_clipped.sum(axis=1))
+    )
+    quantity_spearman = spearmanr(quantity, observed_clipped.sum(axis=1))
+
+    # Print summary
+    print(f"Median Track Pearson: {track_pearson.median():.4f}")
+    print(
+        f"Mean Track Pearson: {track_pearson.mean():.4f} "
+        + f"+/- {track_pearson.std():.4f}"
+    )
+    print(f"Median Track JS Distance: {pd.Series(track_js_distance).median():.4f} ")
+    print(
+        f"Mean Track JS Distance: {pd.Series(track_js_distance).mean():.4f} "
+        + f"+/- {pd.Series(track_js_distance).std():.4f}"
+    )
+    print(f"Track Directionality Pearson: {directionality_pearson[0]:.4f}")
+    print(f"TSS Position Pearson: {tss_pos_pearson[0]:.4f}")
+    print(f"Quantity Log Pearson: {quantity_log_pearson[0]:.4f}")
+    print(f"Quantity Spearman: {quantity_spearman[0]:.4f}")
+
+    # Save metrics
+    with h5py.File(args.output, "w") as hf:
+        hf.create_dataset(
+            "track_pearson", data=track_pearson.to_numpy(), compression="gzip"
+        )
+        hf.create_dataset(
+            "track_js_distance", data=track_js_distance, compression="gzip"
+        )
+        hf.create_dataset(
+            "track_directionality",
+            data=np.array(directionality_pearson),
+            compression="gzip",
+        )
+        hf.create_dataset("tss_pos_pearson", data=tss_pos_pearson, compression="gzip")
+        hf.create_dataset(
+            "quantity_log_pearson",
+            data=np.array(quantity_log_pearson),
+            compression="gzip",
+        )
+        hf.create_dataset(
+            "quantity_spearman", data=np.array(quantity_spearman), compression="gzip"
+        )
+
+
+if __name__ == "__main__":
+    main()

clipnet/_fit_nn.py

@@ -0,0 +1,63 @@
+## NOT IMPLEMENTED
+
+"""
+This script fits a NN model using clipnet.py. It requires a NN architecture file, which
+must contain a function named construct_nn that returns a tf.keras.models.Model object.
+It also requires a dataset_params.py file which specifies parameters and file paths
+associated with the dataset of interest.
+"""
+
+import argparse
+import logging
+import os
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "4"
+logging.getLogger("tensorflow").setLevel(logging.FATAL)
+from . import clipnet
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("model_dir", type=str, help="directory to save models to")
+    parser.add_argument(
+        "--prefix",
+        type=str,
+        default="rnn_v10",
+        help="the prefix of the nn architecture file. Must contain a construct_nn \
+            method, an opt hash that contains all the optimizer hyperparameters, and a \
+            compile_params hash that specifies what loss and metrics are reported. \
+            Models will be saved under this prefix.",
+    )
+    parser.add_argument(
+        "--resume_checkpoint",
+        type=str,
+        default=None,
+        help="resume training from this model.",
+    )
+    parser.add_argument(
+        "--gpu",
+        type=int,
+        default=None,
+        help="Index of GPU to use (starting from 0). If not invoked, uses CPU.",
+    )
+    parser.add_argument(
+        "--n_gpus",
+        type=int,
+        default=0,
+        help="Number of GPUs to use. If not invoked, uses CPU.",
+    )
+    args = parser.parse_args()
+
+    if args.n_gpus > 1:
+        nn = clipnet.CLIPNET(n_gpus=args.n_gpus, prefix=args.prefix)
+    else:
+        nn = (
+            clipnet.CLIPNET(n_gpus=1, use_specific_gpu=args.gpu, prefix=args.prefix)
+            if args.gpu is not None
+            else clipnet.CLIPNET(n_gpus=0, prefix=args.prefix)
+        )
+    nn.fit(model_dir=args.model_dir, resume_checkpoint=args.resume_checkpoint)
+
+
+if __name__ == "__main__":
+    main()