dragonnfruit 0.3.0__tar.gz

dragonnfruit-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Jacob Schreiber
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dragonnfruit-0.3.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.1
+Name: dragonnfruit
+Version: 0.3.0
+Summary: dragonnfruit is a method for analyzing scATAC-seq experiments.
+Home-page: https://github.com/jmschrei/dragonnfruit
+Author: Jacob Schreiber
+Author-email: jmschreiber91@gmail.com
+License: LICENSE.txt
+License-File: LICENSE

dragonnfruit-0.3.0/README.md

@@ -0,0 +1,2 @@
+# dragonnfruit
+A method for analyzing scATAC-seq experiments.

dragonnfruit-0.3.0/cmd/dragonnfruit

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# dragonnfruit command-line tool
+# Author: Jacob Schreiber <jmschreiber91@gmail.com>
+
+import sys
+import numpy
+import torch
+import pyfaidx
+import argparse
+
+import deepdish as dd
+
+from dragonnfruit.preprocessing import fragments_to_sparse
+from dragonnfruit.preprocessing import preprocess_atac
+
+import json
+
+desc = """dragonnfruit is an extension of ChromBPNet for single-cell data.
+	At a high level, it takes in nucleotide sequence and some aspect of cell
+	state, such as an LSI matrix from scATAC-seq data or gene expression data
+	from multimodal data, and makes predictions for dynamically pseudobulked
+	scATAC-seq signal. It works by running the cell state representation through
+	a small multi-layer perceptron and outputting the convolution parameters
+	in the accessibility component of a ChromBPNet model. Put another way: 
+	after training a dragonnfruit model, one can produce a ChromBPNet model
+	for each cell in the experiment.""" 
+
+# Read in the arguments
+parser = argparse.ArgumentParser(description=desc)
+subparsers = parser.add_subparsers(help="Must be either 'train', 'predict', 'interpret', or 'marginalize'.", required=True, dest='cmd')
+
+preprocess_help = """Preprocess a set of fragment files, and optionally
+	inclusion/exclusion matrices, into a set of sparse matrices used in training
+	the dragonnfruit model."""
+preprocess_parser = subparsers.add_parser("preprocess", help=preprocess_help)
+preprocess_parser.add_argument("-p", "--parameters", type=str, required=True,
+	help="A JSON file containing the parameters for preprocessing the data.")
+
+
+# Pull the arguments
+args = parser.parse_args()
+
+if args.cmd == "preprocess":
+	with open(args.parameters, "r") as infile:
+		parameters = json.load(infile)
+
+	default_parameters = {
+		'fragments': None,
+		'loci': None,
+		'h5_filename':'dragonnfruit_data.h5',
+		'rd_filename':'dragonnfruit_read_depths.npz',
+		'chrom_sizes': None,
+		'include_cells': None,
+		'exclude_cells': None,
+		'chroms': ['chr1', 'chr2', 'chr3', 'chr4', 'chr5', 'chr6', 'chr7', 
+			'chr8', 'chr9', 'chr10', 'chr11', 'chr12', 'chr13', 'chr14', 
+			'chr15', 'chr16', 'chr17', 'chr18', 'chr19', 'chr20', 'chr21', 
+			'chr22', 'chrX'],
+		'cell_name_prefixes': None,
+		'max_fragment_length': 1000,
+		'start_offset': 4,
+		'end_offset': -5,
+		'verbose': False
+	}
+
+	for parameter, value in default_parameters.items():
+		if parameter not in parameters:
+			if value is None and parameter in ('fragments', 'chrom_sizes'):
+				raise ValueError("Must provide value for '{}'".format(parameter))
+
+			parameters[parameter] = value
+
+	###
+
+	'''
+	X_cscs, read_depths = fragments_to_sparse(
+		fragments=parameters['fragments'],
+		chrom_sizes=parameters['chrom_sizes'],
+		chroms=parameters['chroms'],
+		include_cells=parameters['include_cells'],
+		exclude_cells=parameters['exclude_cells'],
+		cell_name_prefixes=parameters['cell_name_prefixes'],
+		max_fragment_length=parameters['max_fragment_length'],
+		start_offset=parameters['start_offset'],
+		end_offset=parameters['end_offset'],
+		verbose=parameters['verbose']
+	)
+
+	dd.io.save(parameters['h5_filename'], X_cscs, compression='blosc')
+	numpy.savez_compressed(parameters['rd_filename'], read_depths)
+	'''
+
+	X_cscs = dd.io.load(parameters['h5_filename'])
+
+	X_pca, X_umap, neighbors = preprocess_atac(X_cscs, 
+		peaks=parameters['loci'], chroms=parameters['chroms'])
+
+	numpy.savez_compressed("X_pca.npz", X_pca)
+	numpy.savez_compressed("neighbors.npz", neighbors)

dragonnfruit-0.3.0/dragonnfruit/__init__.py

@@ -0,0 +1,4 @@
+# __init__.py
+# Author: Jacob Schreiber <jmschreiber91@gmail.com>
+
+__version__ = '0.3.0'

dragonnfruit-0.3.0/dragonnfruit/attributions.py

@@ -0,0 +1,66 @@
+
+# attributions.py
+# Author: Jacob Schreiber <jmschreiber91@gmail.com>
+
+import numpy
+import torch
+
+from tqdm import trange
+from captum.attr import DeepLiftShap
+
+from bpnetlite.attributions import dinucleotide_shuffle
+from bpnetlite.attributions import hypothetical_attributions
+
+class ProfileWrapper(torch.nn.Module):
+	"""A wrapper class that returns transformed profiles.
+
+	This class takes in a trained model and returns the weighted softmaxed
+	outputs of the first dimension. Specifically, it takes the predicted
+	"logits" and takes the dot product between them and the softmaxed versions
+	of those logits. This is for convenience when using captum to calculate
+	attribution scores.
+	
+	Parameters
+	----------
+	model: torch.nn.Module
+		A torch model to be wrapped.
+	"""
+
+	def __init__(self, model):
+		super(ProfileWrapper, self).__init__()
+		self.model = model
+
+	def forward(self, X, cell_states):
+		logits = self.model(X, cell_states)
+		logits = logits - torch.mean(logits, dim=-1, keepdims=True) 
+		l = torch.clone(logits).detach()
+		
+		y = torch.exp(l - torch.logsumexp(l, dim=-1, keepdims=True))
+		return (logits * y).sum(axis=-1, keepdims=True)
+
+
+def calculate_attributions(model, X, cell_states, n_shuffles=10, batch_size=1):
+	wrapper = ProfileWrapper(model.cuda())
+	ig = DeepLiftShap(wrapper)
+
+	reference = dinucleotide_shuffle(X, n_shuffles=n_shuffles).cuda()
+	reference = reference.type(X.dtype)
+	
+	X = X.unsqueeze(0).cuda()
+	_cell_states = cell_states.cuda()
+
+	attributions = []
+	with torch.no_grad():
+		for start in trange(0, len(cell_states), batch_size):
+			c_ = _cell_states[start:start+batch_size]
+			X_ = X.expand(len(c_), -1, -1)
+			
+			attr = ig.attribute(X_, reference, target=0, 
+				additional_forward_args=(c_,), 
+				custom_attribution_func=hypothetical_attributions)
+
+			attr = (attr * X_).cpu()
+			attributions.append(attr)
+	
+	attributions = torch.cat(attributions)    
+	return attributions

dragonnfruit-0.3.0/dragonnfruit/io.py

@@ -0,0 +1,267 @@
+# io.py
+# Author: Jacob Schreiber <jmschreiber91@gmail.com>
+
+import time
+import numpy
+import scipy
+import torch
+import pandas
+
+from bpnetlite.io import one_hot_encode
+
+
+def _extract_example(self, chrom, mid, cell_idx, idx):
+	start, end = mid - self.window // 2, mid + self.window // 2
+	neighbs = self.neighbors[cell_idx]
+
+	X = self.sequence[chrom][start:end].T.astype('float32')
+	y = self.signal[chrom][:, start+self.trimming:end-self.trimming]
+	y = numpy.array(y[neighbs].sum(axis=0))[0]
+
+	c = self.cell_states[cell_idx]
+	r = self.read_depths[cell_idx]
+
+	if self.reverse_complement and idx % 2 == 0:
+		X = X[::-1][:, ::-1].copy()
+		y = y[::-1].copy()
+
+	X = torch.from_numpy(X)
+	y = torch.from_numpy(y)
+	c = torch.from_numpy(c)
+	r = torch.from_numpy(r)
+	return X, y, c, r
+
+
+class LocusGenerator(torch.utils.data.Dataset):
+	"""A data generator for dragonnfruit inputs. Adapted from bpnet-lite.
+
+	This generator takes in a set of sequences and output signals 
+	and will return a single element with random jitter and reverse-complement 
+	augmentation applied. Because the data is single-cell where the output
+	signals differ across cells, each returned element is a random locus
+	in a random cell. 
+
+	A conceptual difference between this DataGenerator and the one implemented
+	in bpnet-lite is that the bpnet-lite one assumes that you can extract all
+	loci into an array. Here, because there are hundreds of thousands of peaks
+	and potentially thousands of cells, it is actually more efficient to 
+
+	Parameters
+	----------
+	sequences: dict of torch.tensors, shape=(n, 4), dtype=torch.float32
+		A dictionary of the nucleotide sequences to use.
+
+	signals: dict of torch.tensors, shape=(n, n_cells), dtype=torch.float32
+		A dictionary of the cell signals
+
+	loci: str or pandas.DataFrame
+		A set of loci to use.
+	
+	cell_states: 
+	"""
+
+	def __init__(self, sequence, signal, loci_file, neighbors, cell_states, 
+		read_depths, trimming, window, chroms, n_cells_per_locus=1, 
+		reverse_complement=True, max_jitter=128, random_state=None):
+		self.trimming = trimming
+		self.window = window
+		self.chroms = chroms
+		self.reverse_complement = reverse_complement
+		self.max_jitter = max_jitter
+		self.random_state = numpy.random.RandomState(random_state)
+
+		self.signal = {chrom: signal[chrom] for chrom in chroms}
+		self.sequence = {chrom: sequence[chrom] for chrom in chroms}
+		self.neighbors = neighbors
+		self.cell_states = cell_states
+		self.read_depths = read_depths
+		self.n_cells_per_locus = n_cells_per_locus
+
+		if not isinstance(loci_file, list):
+			loci_file = [loci_file]
+
+		loci = []
+		names = ['chrom', 'start', 'end']
+		for filename in loci_file:
+			loci_ = pandas.read_csv(filename, sep='\t', usecols=[0, 1, 2], 
+				header=None, index_col=False, names=names)
+			loci_['mid'] = (loci_['end'] - loci_['start']) // 2 + loci_['start']
+			loci_ = loci_[numpy.isin(loci_['chrom'], chroms)]
+			loci_ = loci_.sample(frac=1, random_state=self.random_state)
+			loci_ = loci_.reset_index(drop=True)
+			loci.append(loci_)
+
+		self.loci = pandas.concat(loci).sort_index().reset_index(drop=True) # interleave
+		print("Loci: {}".format(self.loci.shape[0]))
+
+	def __len__(self):
+		return self.loci.shape[0]
+
+	def __getitem__(self, idx):
+		chrom, _, _, mid = self.loci.iloc[idx // self.n_cells_per_locus]
+		mid += self.random_state.randint(-self.max_jitter, self.max_jitter+1)
+		cell_idx = self.random_state.randint(self.cell_states.shape[0])
+		return _extract_example(self, chrom, mid, cell_idx, idx)
+
+	
+class GWGenerator(torch.utils.data.Dataset):
+	"""A data generator for dragonnfruit inputs. Adapted from bpnet-lite.
+
+	This generator takes in a set of sequences and output signals 
+	and will return a single element with random jitter and reverse-complement 
+	augmentation applied. Because the data is single-cell where the output
+	signals differ across cells, each returned element is a random locus
+	in a random cell. 
+
+	A conceptual difference between this DataGenerator and the one implemented
+	in bpnet-lite is that the bpnet-lite one assumes that you can extract all
+	loci into an array. Here, because there are hundreds of thousands of peaks
+	and potentially thousands of cells, it is actually more efficient to 
+
+	Parameters
+	----------
+	sequences: dict of torch.tensors, shape=(n, 4), dtype=torch.float32
+		A dictionary of the nucleotide sequences to use.
+
+	signals: dict of torch.tensors, shape=(n, n_cells), dtype=torch.float32
+		A dictionary of the cell signals
+
+	loci: str or pandas.DataFrame
+		A set of loci to use.
+	
+	cell_states: 
+	"""
+
+	def __init__(self, sequence, signal, neighbors, cell_states, 
+		read_depths, trimming, window, chroms, cells_per_loci=1, 
+		reverse_complement=True, random_state=None):
+		self.trimming = trimming
+		self.window = window
+		self.chroms = chroms
+		self.cells_per_loci = cells_per_loci
+		self.reverse_complement = reverse_complement
+		self.random_state = numpy.random.RandomState(random_state)
+
+		self.signal = {chrom: signal[chrom] for chrom in chroms}
+		self.sequence = {chrom: sequence[chrom] for chrom in chroms}
+		self.neighbors = neighbors
+		self.cell_states = cell_states
+		self.read_depths = read_depths
+		self._lengths = numpy.array([seq.shape[0] for seq in self.sequence.values()])
+		print(self._lengths)
+
+	def __len__(self):
+		return sum(self._lengths)
+
+	def __getitem__(self, idx):
+		c_idx = numpy.random.choice(len(self._lengths), 
+			p=self._lengths / self._lengths.sum())
+		chrom = self.chroms[c_idx]
+
+		mid = numpy.random.randint(10000, self._lengths[c_idx]-10000)
+		cell_idx = numpy.random.randint(self.cell_states.shape[0])
+		return _extract_example(self, chrom, mid, cell_idx, idx)
+
+
+class GWBGenerator(torch.utils.data.Dataset):
+	"""A data generator for dragonnfruit inputs. Adapted from bpnet-lite.
+
+	This generator takes in a set of sequences and output signals 
+	and will return a single element with random jitter and reverse-complement 
+	augmentation applied. Because the data is single-cell where the output
+	signals differ across cells, each returned element is a random locus
+	in a random cell. 
+
+	A conceptual difference between this DataGenerator and the one implemented
+	in bpnet-lite is that the bpnet-lite one assumes that you can extract all
+	loci into an array. Here, because there are hundreds of thousands of peaks
+	and potentially thousands of cells, it is actually more efficient to 
+
+	Parameters
+	----------
+	sequences: dict of torch.tensors, shape=(n, 4), dtype=torch.float32
+		A dictionary of the nucleotide sequences to use.
+
+	signals: dict of torch.tensors, shape=(n, n_cells), dtype=torch.float32
+		A dictionary of the cell signals
+
+	loci: str or pandas.DataFrame
+		A set of loci to use.
+	
+	cell_states: 
+	"""
+
+	def __init__(self, sequence, signal, neighbors, cell_states, 
+		read_depths, trimming, window, chroms, batch_size=1024, 
+		cells_per_loci=1, reverse_complement=True, random_state=None):
+		self.trimming = trimming
+		self.window = window
+		self.chroms = chroms
+		self.reverse_complement = reverse_complement
+		self.batch_size = batch_size
+		self.cells_per_loci = cells_per_loci
+		self.random_state = numpy.random.RandomState(random_state)
+
+		print(signal['chr21'].dtype, sequence['chr21'].dtype)
+
+		self.signal = {chrom: signal[chrom] for chrom in chroms}
+		self.sequence = {chrom: sequence[chrom] for chrom in chroms}
+		self.neighbors = neighbors
+		self.cell_states = cell_states
+		self.read_depths = read_depths
+		self._lengths = numpy.array([seq.shape[0] for seq in self.sequence.values()])
+		self._probs = self._lengths / self._lengths.sum()
+
+		print(self._lengths)
+
+	def __len__(self):
+		return sum(self._lengths)
+
+	def __getitem__(self, idx):
+		tic = time.time()
+		X = numpy.empty((self.batch_size, 4, self.window), dtype=numpy.int8)
+		y = numpy.empty((self.batch_size, self.window - self.trimming*2), dtype=numpy.float32)
+		c = numpy.empty((self.batch_size, self.cell_states.shape[1]), dtype=numpy.float32)
+		r = numpy.empty((self.batch_size, 1), dtype=numpy.float32)
+
+		cell_idxs = self.random_state.randint(self.cell_states.shape[0], size=self.batch_size)
+		chrom_idxs = self.random_state.choice(len(self._lengths), p=self._probs, size=self.batch_size) 
+
+		nc = self.cells_per_loci
+		a = time.time() - tic
+		b, d, e = 0, 0, 0
+
+		for i, (cell_idx, chrom_idx) in enumerate(zip(cell_idxs, chrom_idxs)):
+			tic = time.time()
+			if i % nc == 0:
+				chrom = self.chroms[chrom_idx]
+
+				mid = self.random_state.randint(10000, self._lengths[chrom_idx]-10000)
+				start, end = mid - self.window // 2, mid + self.window // 2
+
+				lidx = (i // nc) * nc
+				X[lidx:lidx+nc] = self.sequence[chrom][start:end].T
+				y_ = self.signal[chrom][:, start+self.trimming:end-self.trimming].tocsr()
+				b += time.time() - tic
+
+			tic = time.time()
+			neighbs = self.neighbors[cell_idx]
+			y[i] = numpy.array(y_[neighbs].sum(axis=0))[0]
+			d += time.time() - tic
+
+			tic = time.time()
+			c[i] = self.cell_states[cell_idx]
+			r[i] = self.read_depths[cell_idx]
+			e += time.time() - tic
+
+			if self.reverse_complement and idx % 2 == 0:
+				X[i] = X[i][::-1][:, ::-1].copy()
+				y[i] = y[i][::-1].copy()
+
+		tic = time.time()
+		X = torch.from_numpy(X.astype('float32'))
+		y = torch.from_numpy(y)
+		c = torch.from_numpy(c)
+		r = torch.from_numpy(r)
+		#print(idx, a, b, d, e, time.time() - tic)
+		return X, y, c, r