biopipen 0.17.6__py3-none-any.whl → 0.18.0__py3-none-any.whl

biopipen/scripts/tcr/TESSA.R

@@ -0,0 +1,198 @@
+source("{{biopipen_dir}}/utils/misc.R")
+
+library(glue)
+library(dplyr)
+library(tidyr)
+library(immunarch)
+library(Seurat)
+library(ggplot2)
+library(ggprism)
+
+immfile <- {{in.immdata | r}}
+exprfile <- {{in.srtobj | r}}
+outfile <- {{out.outfile | r}}
+python <- {{envs.python | r}}
+within_sample <- {{envs.within_sample | r}}
+assay <- {{envs.assay | r}}
+predefined_b <- {{envs.predefined_b | r}}
+max_iter <- {{envs.max_iter | int}}
+tessa_srcdir <- "{{biopipen_dir}}/scripts/tcr/TESSA_source"
+
+outdir <- dirname(outfile)
+result_dir <- file.path(outdir, "result")
+tessa_dir <- file.path(outdir, "tessa")
+if (!dir.exists(result_dir)) dir.create(result_dir)
+if (!dir.exists(tessa_dir)) dir.create(tessa_dir)
+
+### Start preparing input files for TESSA
+# Prepare input files
+print("Preparing TCR input file ...")
+immdata <- readRDS(immfile)
+
+has_VJ <- "V.name" %in% colnames(immdata$data[[1]]) && "J.name" %in% colnames(immdata$data[[1]])
+# Merge all samples
+tcrdata <- do_call(rbind, lapply(seq_len(nrow(immdata$meta)), function(i) {
+    # Clones  Proportion   CDR3.aa                       Barcode
+    # 5      4 0.008583691 CAVRDTGNTPLVF;CASSEYSNQPQHF   GTTCGGGCACTTACGA-1;TCTCTAAGTACCAGTT-1
+    # 6      4 0.008583691 CALTQAAGNKLTF;CASRPEDLRGQPQHF GCTTGAAGTCGGCACT-1;TACTCGCTCCTAAGTG-1
+    if (has_VJ) {
+        cldata = immdata$data[[i]][, c("Barcode", "CDR3.aa", "V.name", "J.name")]
+    } else {
+        cldata = immdata$data[[i]][, c("Barcode", "CDR3.aa")]
+    }
+    # # A tibble: 4 × 5
+    # Sample                  Patient     Timepoint Tissue
+    # <chr>                   <chr>       <chr>     <chr>
+    # 1 MC1685Pt011-Baseline-PB MC1685Pt011 Baseline  PB
+    mdata = as.list(immdata$meta[i, , drop=FALSE])
+    for (mname in names(mdata)) {
+        assign(mname, mdata[[mname]])
+    }
+
+    cldata %>%
+        separate_rows(Barcode, sep=";") %>%
+        # Just in case there are duplicated barcodes
+        distinct(Barcode, .keep_all = TRUE) %>%
+        mutate(Barcode = glue("{{envs.prefix}}{Barcode}"), sample = Sample)
+}))
+if (has_VJ) {
+    tcrdata <- tcrdata %>% dplyr::mutate(
+        v_gene = sub("-\\d+$", "", V.name),
+        j_gene = sub("-\\d+$", "", J.name)
+    ) %>% dplyr::select(
+        contig_id = Barcode,
+        cdr3 = CDR3.aa,
+        v_gene,
+        j_gene,
+        sample
+    )
+} else {
+    tcrdata <- tcrdata %>% dplyr::select(
+        contig_id = Barcode,
+        cdr3 = CDR3.aa,
+        sample
+    )
+}
+
+
+print("Preparing expression input file ...")
+is_seurat <- endsWith(tolower(exprfile), ".rds")
+is_gz <- endsWith(tolower(exprfile), ".gz")
+
+if (is_seurat) {
+    sobj <- readRDS(exprfile)
+    expr <- GetAssayData(sobj, slot = "data", assay = assay)
+} else if (is_gz) {
+    expr <- read.table(gzfile(exprfile), sep="\t", header=TRUE, row.names=1)
+} else {
+    expr <- read.table(exprfile, sep="\t", header=TRUE, row.names=1)
+}
+
+cell_ids <- intersect(tcrdata$contig_id, colnames(expr))
+# Warning about unused cells
+unused_tcr_cells <- setdiff(tcrdata$contig_id, cell_ids)
+unused_expr_cells <- setdiff(colnames(expr), cell_ids)
+if (length(unused_tcr_cells) > 0) {
+    warning(glue("{length(unused_tcr_cells)}/{nrow(tcrdata)} TCR cells are not used."), immediate. = TRUE)
+}
+if (length(unused_expr_cells) > 0) {
+    warning(glue("{length(unused_expr_cells)}/{ncol(expr)} expression cells are not used."), immediate. = TRUE)
+}
+if (length(cell_ids) == 0) {
+    stop("No common cells between TCR and expression data. Are you using the correct prefix?")
+}
+tcrdata <- tcrdata[tcrdata$contig_id %in% cell_ids, , drop=FALSE]
+expr <- as.matrix(expr)[, tcrdata$contig_id, drop=FALSE]
+
+# Write input files
+print("Writing input files ...")
+write.table(tcrdata, file.path(tessa_dir, "tcrdata.txt"), sep=",", quote=FALSE, row.names=FALSE)
+write.table(expr, file.path(tessa_dir, "exprdata.txt"), sep=",", quote=FALSE, row.names=TRUE, col.names=TRUE)
+
+### End preparing input files for TESSA
+
+### Start running TESSA
+print("Running TESSA ...")
+
+# The original TESSA uses a python wrapper to run the encoder and tessa model
+# here we run those two steps directly here
+
+print("- Running encoder ...")
+cmd_encoder <- paste(
+    python,
+    file.path(tessa_srcdir, "BriseisEncoder.py"),
+    "-tcr",
+    file.path(tessa_dir, "tcrdata.txt"),
+    "-model",
+    file.path(tessa_srcdir, "TrainedEncoder.h5"),
+    "-embeding_vectors",
+    file.path(tessa_srcdir, "Atchley_factors.csv"),
+    "-output_TCR",
+    file.path(tessa_dir, "tcr_encoded.txt"),
+    "-output_log",
+    file.path(tessa_dir, "tcr_encoder.log")
+)
+if (has_VJ) {
+    cmd_encoder <- paste(
+        cmd_encoder,
+        "-output_VJ",
+        file.path(tessa_dir, "tcr_vj.txt")
+    )
+}
+print(paste("- ", cmd_encoder))
+
+rc <- system(cmd_encoder)
+if (rc != 0) {
+    stop("Error: Failed to run encoder.")
+}
+
+print("- Running TESSA model ...")
+source(file.path(tessa_srcdir, "real_data.R"))
+
+tessa <- run_tessa(
+    tessa_srcdir,
+    file.path(tessa_dir, "exprdata.txt"),
+    file.path(tessa_dir, "tcr_encoded.txt"),
+    file.path(tessa_dir, "tcrdata.txt"),
+    result_dir,
+    within_sample,
+    (if (!predefined_b) NULL else file.path(tessa_srcdir, "fixed_b.csv")),
+    max_iter = max_iter
+)
+
+# Save TESSA results
+print("Saving TESSA results ...")
+if (is_seurat) {
+    cells <- rownames(sobj@meta.data)
+    sobj@meta.data <- sobj@meta.data %>%
+        mutate(
+            TESSA_Cluster = tessa$meta[
+                match(cells, tessa$meta$barcode),
+                "cluster_number"
+            ]
+        ) %>%
+        add_count(TESSA_Cluster, name = "TESSA_Cluster_Size")
+    rownames(sobj@meta.data) <- cells
+    saveRDS(sobj, outfile)
+} else {
+    out <- tessa$meta %>%
+        dplyr::select(barcode, TESSA_Cluster = cluster_number) %>%
+        add_count(TESSA_Cluster, name = "TESSA_Cluster_Size")
+    write.table(out, outfile, sep="\t", quote=FALSE, row.names=FALSE, col.names=TRUE)
+}
+
+# Post analysis
+print("Post analysis ...")
+plot_tessa(tessa, result_dir)
+plot_Tessa_clusters(tessa, result_dir)
+
+p <- tessa$meta %>%
+    dplyr::select(barcode, TESSA_Cluster = cluster_number) %>%
+    add_count(TESSA_Cluster, name = "TESSA_Cluster_Size") %>%
+    ggplot(aes(x = TESSA_Cluster_Size)) +
+    geom_histogram(binwidth = 1) +
+    theme_prism()
+
+png(file.path(result_dir, "Cluster_size_dist.png"), width=8, height=8, units="in", res=100)
+print(p)
+dev.off()

biopipen/scripts/tcr/TESSA_source/Atchley_factors.csv

@@ -0,0 +1,21 @@
+Amino acid,Factor I,Factor II,Factor III,Factor IV,Factor V
+A,-0.591,-1.302,-0.733,1.57,-0.146
+C,-1.343,0.465,-0.862,-1.02,-0.255
+D,1.05,0.302,-3.656,-0.259,-3.242
+E,1.357,-1.453,1.477,0.113,-0.837
+F,-1.006,-0.59,1.891,-0.397,0.412
+G,-0.384,1.652,1.33,1.045,2.064
+H,0.336,-0.417,-1.673,-1.474,-0.078
+I,-1.239,-0.547,2.131,0.393,0.816
+K,1.831,-0.561,0.533,-0.277,1.648
+L,-1.019,-0.987,-1.505,1.266,-0.912
+M,-0.663,-1.524,2.219,-1.005,1.212
+N,0.945,0.828,1.299,-0.169,0.933
+P,0.189,2.081,-1.628,0.421,-1.392
+Q,0.931,-0.179,-3.005,-0.503,-1.853
+R,1.538,-0.055,1.502,0.44,2.897
+S,-0.228,1.399,-4.76,0.67,-2.647
+T,-0.032,0.326,2.213,0.908,1.313
+V,-1.337,-0.279,-0.544,1.242,-1.262
+W,-0.595,0.009,0.672,-2.128,-0.184
+Y,0.26,0.83,3.097,-0.838,1.512

biopipen/scripts/tcr/TESSA_source/BriseisEncoder.py

@@ -0,0 +1,168 @@
+"""
+##########################################################################################################################
+# BriseisEncoder 1.0.0: A deep learning encoder for CDR3 sequences in the TCR space
+##########################################################################################################################
+# BriseisEncoder is capable of transforming CDR3 peptide sequences from productive
+# TCR-beta chains into 15-digit informative numerical vectors.
+# 01312019 Ze Zhang <Ze.Zhang@UTsouthwestern.edu>
+##########################################################################################################################
+# Dependencies:
+# Python 3.6.4 (preferred)
+# numpy (>=1.15.4), pandas (>=0.23.4), keras (>=2.2.4)
+##########################################################################################################################
+# Parameters:
+# tcr_dir: a .csv file to input the CDR3 sequences, contains at least 2 columns naming 'contig_id' and 'cdr3'. Two optional
+# 		  columns 'v_gene' and 'j_gene' denotes the V gene and J gene subgroups (TRBV1-30 and TRBJ1/2) of TRB recombinants.
+# 		  'contig_id' should contains non-repeated ID strings, and 'cdr3' should contains valid TRB CDR3 peptide sequences.
+# model_dir: a .h5 file containing trained encoding models from CDR3 seqs called from bulk-tumor RNA-seq data.
+# aa_dict_dir: a .csv file storing Atchley's amino acid embedding, details in https://www.ncbi.nlm.nih.gov/pubmed/15851683.
+# output_encodedTCR_dir: a .csv file dir to store encoded CDR3 peptide seqs.
+# output_log_dir: a plain text log-file to record any errors or warnings from this script.
+# output_encodedVJ_dir: a .csv file dir to store one-hot encoded TRBV/TRBJ genes.
+##########################################################################################################################
+# Example:
+# python3 BriseisEncoder.py -tcr TestCDR3.csv -model Trained_encoder.h5 -embeding_vectors Atchley_factors.csv \
+# -output_TCR test.csv -output_VJ testVJ.csv -output_log test.log
+##########################################################################################################################
+"""  # noqa: E501
+
+# Import dependencies
+import sys
+
+# sys.path.append('/home2/s421955/.conda/envs/keras_test/site-packages')
+import numpy as np
+import pandas as pd
+import os
+import csv
+from keras.models import load_model
+from keras.models import Model
+
+# Read data
+args = sys.argv
+tcr_dir = args[args.index("-tcr") + 1]
+model_dir = args[args.index("-model") + 1]
+aa_dict_dir = args[args.index("-embeding_vectors") + 1]
+output_encodedTCR_dir = args[args.index("-output_TCR") + 1]
+if "-output_VJ" in args:
+    output_encodedVJ_dir = args[args.index("-output_VJ") + 1]
+output_log_dir = args[args.index("-output_log") + 1]
+encode_dim = 80
+
+
+# Define functions
+def preprocess(filedir):
+    # Preprocess TCR files
+    print("Processing: " + filedir)
+    if not os.path.exists(filedir):
+        print("Invalid file path: " + filedir)
+        return 0
+    dataset = pd.read_csv(filedir, header=0)
+    if dataset.isnull().values.any():
+        print("Input data contains NAs.")
+        # dataset = dataset.dropna()
+    data_new = pd.DataFrame(
+        {
+            "contig_id": dataset["contig_id"],
+            "cdr3": dataset["cdr3"],
+            "v_gene": None,
+            "j_gene": None,
+        }
+    )
+    if "v_gene" in dataset.columns:
+        data_new["v_gene"] = dataset["v_gene"]
+    else:
+        data_new = data_new.drop(columns="v_gene")
+    if "j_gene" in dataset.columns:
+        data_new["j_gene"] = dataset["j_gene"]
+    else:
+        data_new = data_new.drop(columns="j_gene")
+    data_new.index = range(0, dataset.shape[0])
+    return data_new
+
+
+def aamapping(peptideSeq, aa_dict, encode_dim):
+    # Transform aa seqs to Atchley's factors.
+    peptideArray = []
+    if len(peptideSeq) > encode_dim:
+        print("Length: " + str(len(peptideSeq)) + " over bound!")
+        peptideSeq = peptideSeq[0:encode_dim]
+    for aa_single in peptideSeq:
+        try:
+            peptideArray.append(aa_dict[aa_single])
+        except KeyError:
+            print("Not proper aaSeqs: " + peptideSeq)
+            peptideArray.append(np.zeros(5, dtype="float64"))
+    for i in range(0, encode_dim - len(peptideSeq)):
+        peptideArray.append(np.zeros(5, dtype="float64"))
+    return np.asarray(peptideArray)
+
+
+def datasetMap(dataset, aa_dict, encode_dim):
+    # Wrapper of aamapping
+    TCR_dict = dict()
+    for i in range(0, len(dataset["cdr3"])):
+        TCR_key = dataset["contig_id"][i]
+        if TCR_key in TCR_dict.keys():
+            TCR_key = TCR_key + "_" + str(i)
+        TCR_dictarray = aamapping(dataset["cdr3"][i], aa_dict, encode_dim)
+        TCR_dict[TCR_key] = TCR_dictarray
+    return TCR_dict
+
+
+def embedVJ(genelist, maplist):
+    # Embed VJgenes
+    VJ_array = []
+    for gene in genelist:
+        ind = np.zeros(len(maplist))
+        try:
+            find = maplist.index(gene)
+            ind[find] = 1
+            VJ_array.append(ind)
+        except ValueError:
+            print("Gene out of bound!" + str(gene))
+            VJ_array.append(ind)
+            next
+    return np.asarray(VJ_array)
+
+
+# Main functions, data preprocess
+log_file = open(output_log_dir, "w")
+sys.stdout = log_file
+print("Mission loading.")
+tcr = preprocess(tcr_dir)
+aa_dict = dict()
+with open(aa_dict_dir, "r") as aa:
+    aa_reader = csv.reader(aa)
+    next(aa_reader, None)
+    for rows in aa_reader:
+        aa_name = rows[0]
+        aa_factor = rows[1 : len(rows)]
+        aa_dict[aa_name] = np.asarray(aa_factor, dtype="float")
+TCR_dict = datasetMap(tcr, aa_dict, encode_dim)
+TCR_contigs = np.stack(list(TCR_dict.values()))
+TCR_contigs = TCR_contigs.reshape(-1, encode_dim, 5, 1)
+# Model prediction
+TCRencoder = load_model(model_dir)
+encoder = Model(TCRencoder.input, TCRencoder.layers[-12].output)
+encoded_mat = encoder.predict(TCR_contigs)
+encoded_mat = pd.DataFrame(encoded_mat, index=tcr["contig_id"])
+encoded_mat.to_csv(output_encodedTCR_dir, sep=",")
+if "v_gene" in tcr.columns or "j_gene" in tcr.columns:
+    maplist = ["TRBV" + str(i) for i in range(1, 31)]
+    maplist.append("TRBJ1")
+    maplist.append("TRBJ2")
+    if "v_gene" in tcr.columns:
+        v_map = embedVJ(tcr["v_gene"], maplist)[:, 0:30]
+    else:
+        print("V genes are missing!")
+        v_map = np.zeros((tcr.shape[0], 30), dtype="float64")
+    if "j_gene" in tcr.columns:
+        j_map = embedVJ(tcr["j_gene"], maplist)[:, 30:32]
+    else:
+        print("J genes are missing!")
+        j_map = np.zeros((tcr.shape[0], 2), dtype="float64")
+    VJ_map = np.concatenate((v_map, j_map), axis=1)
+    VJ_map = pd.DataFrame(data=VJ_map, index=tcr["contig_id"], columns=maplist)
+    VJ_map.to_csv(output_encodedVJ_dir, sep=",")
+print("Mission Accomplished.\n")
+log_file.close()

biopipen/scripts/tcr/TESSA_source/MCMC_control.R

@@ -0,0 +1,71 @@
+Tessa<-function(e,cdr3,t,hyper_priors,max_iter,sample_id=NULL,save=NULL,b=NULL,seed_num=123)
+{
+  # initialization
+  cat('\nInitialization\n')
+  options(warn=2)
+  if ((!is.null(save)) && (!file.exists(save))) {dir.create(save)}
+  if (!is.null(sample_id)){
+    if(length(sample_id)!=ncol(t)){
+      print('Unmatched sample IDs!')
+      break
+    }
+  }
+  if(!is.null(b)){
+    preset_b=TRUE
+  }else{
+    preset_b=FALSE
+  }
+  print(seed_num)
+  initialized=initialize(t,cdr3,e,hyper_priors,sample_id,b,seed_num=seed_num)
+  t=initialized$t;meta=initialized$meta;a=initialized$a;b=initialized$b;sigma=initialized$sigma;
+  K=initialized$K;ak=initialized$ak;phi=initialized$phi;t0=initialized$t0;
+  master_dist_e=initialized$master_dist_e;de=initialized$de;dt=initialized$dt
+
+  lambda=hyper_priors$lambda;xi=hyper_priors$xi;g=hyper_priors$g
+  tau=hyper_priors$tau;u=hyper_priors$u;v=hyper_priors$v;
+  initialize_cluster_factor=hyper_priors$initialize_cluster_factor
+
+  # some intermediate variables for computational efficiencies
+  meta_dedup=meta[!duplicated(meta$group_ID),] # meta: cell level; meta_dedup: group level
+  rownames(meta_dedup)=meta_dedup$group_ID
+  meta_dedup=meta_dedup[colnames(t),]
+  updated_recent=c() # for recording acceptance rates of b
+  mean_t=rowMeans(t)
+
+  # MCMC
+  for (iter in 1:max_iter)
+  {
+    # report
+    cat(paste('\nIteration round:',iter,"\n"))
+    cat(paste("  # clusters:",K,"\n"))
+    cluster_rate=1-sum(sapply(dt,function(x) length(x)==1))/length(unlist(dt))
+    cat(paste("  Clustering rate:",round(cluster_rate,d=3),"\n"))
+
+    # Dirichlet process
+    results=DP(meta_dedup,meta,t0,dt,de,ak,phi,t,lambda,g,sigma,b,master_dist_e,K,
+      a,xi,mean_t,sample_id)
+    t0=results$t0;phi=results$phi;de=results$de;dt=results$dt;ak=results$ak;
+    c=results$c;K=results$K;meta_dedup$cluster_number=results$cluster_number
+
+    # other parameters
+    t0=update_t0(t,meta_dedup,K,lambda,b,phi,t0)
+    ak=update_ak(K,dt,de,sigma,a,g,phi,ak)
+    regression_loss=sapply(1:K,function(k) sum((de[[k]]-ak[k]*dt[[k]])^2))
+    sigma=update_sigma(u,v,K,phi,g,ak,a,de,dt,regression_loss)
+    a=update_a(tau,K,g,sigma,ak)
+    b_result=update_b(u,v,phi,t,t0,meta_dedup,K,dt,de,ak,sigma,regression_loss,b,preset_b)
+    b=b_result$b;dt=b_result$dt
+    updated_recent=c(b_result$updated,updated_recent)
+    if (length(updated_recent)>200) {updated_recent=updated_recent[1:200]}
+    cat(paste("  Recent b acceptance rate:",round(sum(updated_recent)/length(updated_recent),d=2),"\n"))
+  }
+
+  # wrap up
+  options(warn=0)
+  meta$cluster_number=meta_dedup[meta$group_ID,"cluster_number"]
+  tessa_results=list(b=b,meta=meta,master_dist_e=master_dist_e,a=a,ak=ak,sigma=sigma,dt=dt,de=de,
+    t=t,meta_dedup=meta_dedup,phi=phi,K=K)
+  save(tessa_results,file=paste(save,"/tessa_final.RData",sep=""))
+  write.csv(meta,file=paste(save,"/result_meta.csv",sep=""),quote=F,row.names=FALSE)
+  return(tessa_results)
+}

biopipen/scripts/tcr/TESSA_source/fixed_b.csv

@@ -0,0 +1,31 @@
+b
+0.104086935938378
+0.108780692900858
+0.0570899431292494
+0.0223191901369765
+0.0625734040149305
+0.0724461005365275
+0.0570765486616299
+0.0735009762613172
+0.0493248771695967
+0.0644907300641909
+0.0505856876380108
+0.0362054942161984
+0.0430213209688376
+0.0338085552633352
+0.0577835864812993
+0.0360238874526061
+0.0497979998974232
+0.0501341494361042
+0.0974383828919103
+0.0950533850505816
+0.0230444436289012
+0.0830927914655238
+0.0632701497792147
+0.0341922249498399
+0.0188572387690941
+0.0220431669369226
+0.0353984720393545
+0.0327647439111161
+0.0298263126422095
+0.0244369519533193

biopipen/scripts/tcr/TESSA_source/initialization.R

@@ -0,0 +1,120 @@
+# find center of each cluster
+find_center<-function(t_cluster,b)
+{
+  names(which.min(colSums((t_cluster-rowMeans(t_cluster))^2/b)))[1]
+}
+
+# to initialize the assignment of clusters. A good initialization will save a lot of MCMC cycles
+initialize_cluster<-function(meta,t,factor=4,sample_id_dedup) # larger factor, less number of clusters
+{
+  t0=t[,meta$group_ID]
+  if(is.null(sample_id_dedup)){
+    k0=round(dim(t0)[2]/factor)
+    if(k0<1){k0=1}
+    cluster_n=cutree(hclust(dist(t(t0))),k=k0)
+  }else{
+    sample_id=sample_id_dedup[meta$group_ID]
+    cluster_n=rep(0,ncol(t0))
+    names(cluster_n)=meta$group_ID
+    for(s in 1:length(unique(sample_id))){
+      sample_id_tmp=unique(sample_id)[s]
+      t0_tmp=t0[,sample_id==sample_id_tmp]
+      k0=round(dim(t0_tmp)[2]/(factor*length(unique(sample_id))))
+      if(k0<1){k0=1}
+      cluster_n_tmp=cutree(hclust(dist(t(t0_tmp))),k=k0)
+      cluster_n_tmp=cluster_n_tmp+max(cluster_n)
+      cluster_n[sample_id==sample_id_tmp]=cluster_n_tmp
+    }
+  }
+  cluster=rep("",length(cluster_n))
+  b_dummy=rep(1,dim(t)[1])
+
+  for (i in unique(cluster_n))
+    {cluster[cluster_n==i]=find_center(t0[,cluster_n==i,drop=F],b_dummy)}
+  cluster
+}
+
+# initialization
+initialize<-function(t,cdr3,e,hyper_priors,sample_id,b,seed_num=123)
+{
+  # setting up
+  set.seed(as.numeric(seed_num))
+  lambda=hyper_priors$lambda;xi=hyper_priors$xi;g=hyper_priors$g
+  tau=hyper_priors$tau;u=hyper_priors$u;v=hyper_priors$v;
+  initialize_cluster_factor=hyper_priors$initialize_cluster_factor
+
+  # preprocess t
+  if(!is.null(sample_id)){
+    cdr3=paste(cdr3,sample_id,sep=';')
+    sample_id_dedup=sample_id[!duplicated(cdr3)]
+    names(sample_id_dedup)=cdr3[!duplicated(cdr3)]
+  }else{
+    sample_id_dedup=NULL
+  }
+  t=t[,!duplicated(cdr3),drop=F]
+  colnames(t)=cdr3[!duplicated(cdr3)]
+
+  # meta object
+  # group ID is just the CDR3 sequence, cluster ID is the CDR3 sequence of the centers
+  meta=data.frame(barcode=colnames(e),group_ID=cdr3,cluster_number=NA,stringsAsFactors = F)
+  meta$cluster_number=initialize_cluster(meta,t,initialize_cluster_factor,sample_id_dedup)
+
+  ## initialize random/placeholder variables
+  # simple ones
+  # the initialization of b is tricky
+  # note this is not sampling from its distribution, but it is ok for initialization
+  if(is.null(b)){
+    b=apply(t,1,var)/10
+  }
+  #b[]=mean(b)
+  K=length(unique(meta$cluster_number))
+
+  # phi
+  phi0=aggregate(meta$group_ID,by=list(meta$cluster_number),function(x) length(unique(x)))
+  phi=phi0[,2]
+  names(phi)=phi0[,1]
+
+  # t0
+  # this is not exactly "right", but not wrong, either
+  t00=aggregate(t(t[,meta$group_ID]),by=list(meta$cluster_number),mean)
+  t0=sapply(1:dim(t00)[1],function(i) as.numeric(unlist(t00[i,-1])),simplify=F)
+  names(t0)=t00$Group.1
+  t0=t0[names(phi)]
+
+  # de, dt
+  tmp=as.matrix(dist(t(e)))
+  tmp=aggregate(tmp,by=list(meta$group_ID),mean)
+  rownames(tmp)=tmp[,1]
+  tmp=as.matrix(tmp[,-1])
+  tmp=aggregate(t(tmp),by=list(meta$group_ID),mean)
+  rownames(tmp)=tmp[,1]
+  master_dist_e=as.matrix(tmp[,-1])
+  master_dist_e=master_dist_e[colnames(t),colnames(t)]
+
+  dt=de=list()
+  coefs=c()
+
+  for(k in 1:K)
+  {
+    c=names(phi)[k]
+    group=unique(meta$group_ID[meta$cluster_number==c])
+    de[[c]]=named_c(NULL,master_dist_e[names(phi)[k],group],group)
+    dt[[c]]=named_c(NULL,colSums((t[,group,drop=F]-t[,c])^2/b/2),group)
+    coefs=c(coefs,coef(lm(de[[c]]~dt[[c]]))[2]) # for estimating good values of a and ak
+  }
+
+  # sigma, this is also tricky
+  sigma=mean(sapply(1:K,function(k) sd(de[[k]])),na.rm=T)*5
+  if (is.na(sigma)) {sigma=1}
+
+  # a, ak
+  # note this is not sampling from its distribution
+  coefs=coefs[which(coefs>0)]
+  a=ifelse(is.na(mean(coefs)),1,mean(coefs))
+  ak=rnorm(K,a,ifelse(is.na(sd(coefs)),1,sd(coefs)))
+  names(ak)=names(phi)
+
+  # return
+  return(list(t=t,meta=meta,a=a,b=b,sigma=sigma,K=K,
+    ak=ak,phi=phi,t0=t0,master_dist_e=master_dist_e,de=de,dt=dt))
+}