smftools 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

smftools/informatics/helpers/ohe_batching.py

@@ -0,0 +1,76 @@
+import os
+import anndata as ad
+import numpy as np
+import concurrent.futures
+from .one_hot_encode import one_hot_encode
+
+def encode_sequence(args):
+    """Parallel helper function for one-hot encoding."""
+    read_name, seq, device = args
+    try:
+        one_hot_matrix = one_hot_encode(seq, device)
+        return read_name, one_hot_matrix
+    except Exception:
+        return None  # Skip invalid sequences
+
+def encode_and_save_batch(batch_data, tmp_dir, prefix, record, batch_number):
+    """Encodes a batch and writes to disk immediately."""
+    batch = {read_name: matrix for read_name, matrix in batch_data if matrix is not None}
+
+    if batch:
+        save_name = os.path.join(tmp_dir, f'tmp_{prefix}_{record}_{batch_number}.h5ad')
+        tmp_ad = ad.AnnData(X=np.zeros((1, 1)), uns=batch)  # Placeholder X
+        tmp_ad.write_h5ad(save_name)
+        return save_name
+    return None
+
+def ohe_batching(base_identities, tmp_dir, record, prefix='', batch_size=100000, progress_bar=None, device='auto', threads=None):
+    """
+    Efficient version of ohe_batching: one-hot encodes sequences in parallel and writes batches immediately.
+
+    Parameters:
+        base_identities (dict): Dictionary mapping read names to sequences.
+        tmp_dir (str): Directory for storing temporary files.
+        record (str): Record name.
+        prefix (str): Prefix for file naming.
+        batch_size (int): Number of reads per batch.
+        progress_bar (tqdm instance, optional): Shared progress bar.
+        device (str): Device for encoding.
+        threads (int, optional): Number of parallel workers.
+
+    Returns:
+        list: List of valid H5AD file paths.
+    """
+    threads = threads or os.cpu_count()  # Default to max available CPU cores
+    batch_data = []
+    batch_number = 0
+    file_names = []
+
+    # Step 1: Prepare Data for Parallel Encoding
+    encoding_args = [(read_name, seq, device) for read_name, seq in base_identities.items() if seq is not None]
+
+    # Step 2: Parallel One-Hot Encoding using threads (to avoid nested processes)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
+        for result in executor.map(encode_sequence, encoding_args):
+            if result:
+                batch_data.append(result)
+
+                if len(batch_data) >= batch_size:
+                    # Step 3: Process and Write Batch Immediately
+                    file_name = encode_and_save_batch(batch_data.copy(), tmp_dir, prefix, record, batch_number)
+                    if file_name:
+                        file_names.append(file_name)
+
+                    batch_data.clear()
+                    batch_number += 1
+
+                if progress_bar:
+                    progress_bar.update(1)
+
+    # Step 4: Process Remaining Batch
+    if batch_data:
+        file_name = encode_and_save_batch(batch_data, tmp_dir, prefix, record, batch_number)
+        if file_name:
+            file_names.append(file_name)
+
+    return file_names

smftools/informatics/helpers/ohe_layers_decode.py

@@ -0,0 +1,32 @@
+# ohe_layers_decode
+
+def ohe_layers_decode(adata, obs_names):
+    """
+    Takes an anndata object and a list of observation names. Returns a list of sequence strings for the reads of interest.
+    Parameters:
+        adata (AnnData): An anndata object.
+        obs_names (list): A list of observation name strings to retrieve sequences for.
+
+    Returns:
+        sequences (list of str): List of strings of the one hot encoded array
+    """
+    import anndata as ad
+    import numpy as np
+    from .ohe_decode import ohe_decode
+
+    # Define the mapping of one-hot encoded indices to DNA bases
+    mapping = ['A', 'C', 'G', 'T', 'N']
+
+    ohe_layers = [f"{base}_binary_encoding" for base in mapping]
+    sequences = []
+
+    for obs_name in obs_names:
+        obs_subset = adata[obs_name]
+        ohe_list = []
+        for layer in ohe_layers:
+            ohe_list += list(obs_subset.layers[layer])
+        ohe_array = np.array(ohe_list)
+        sequence = ohe_decode(ohe_array)
+        sequences.append(sequence)
+        
+    return sequences

smftools/informatics/helpers/one_hot_decode.py

@@ -0,0 +1,27 @@
+# one_hot_decode
+
+# String encodings
+def one_hot_decode(ohe_array):
+    """
+    Takes a flattened one hot encoded array and returns the sequence string from that array.
+    Parameters:
+        ohe_array (np.array): A one hot encoded array
+
+    Returns:
+        sequence (str): Sequence string of the one hot encoded array
+    """
+    import numpy as np
+    # Define the mapping of one-hot encoded indices to DNA bases
+    mapping = ['A', 'C', 'G', 'T', 'N']
+    
+    # Reshape the flattened array into a 2D matrix with 5 columns (one for each base)
+    one_hot_matrix = ohe_array.reshape(-1, 5)
+    
+    # Get the index of the maximum value (which will be 1) in each row
+    decoded_indices = np.argmax(one_hot_matrix, axis=1)
+    
+    # Map the indices back to the corresponding bases
+    sequence_list = [mapping[i] for i in decoded_indices]
+    sequence = ''.join(sequence_list)
+    
+    return sequence

smftools/informatics/helpers/one_hot_encode.py

@@ -0,0 +1,57 @@
+# one_hot_encode
+
+def one_hot_encode(sequence, device='auto'):
+    """
+    One-hot encodes a DNA sequence.
+
+    Parameters:
+        sequence (str or list): DNA sequence (e.g., "ACGTN" or ['A', 'C', 'G', 'T', 'N']).
+
+    Returns:
+        ndarray: Flattened one-hot encoded representation of the input sequence.
+    """
+    import numpy as np
+
+    mapping = np.array(['A', 'C', 'G', 'T', 'N'])
+
+    # Ensure input is a list of characters
+    if not isinstance(sequence, list):
+        sequence = list(sequence)  # Convert string to list of characters
+
+    # Handle empty sequences
+    if len(sequence) == 0:
+        print("Warning: Empty sequence encountered in one_hot_encode()")
+        return np.zeros(len(mapping))  # Return empty encoding instead of failing
+
+    # Convert sequence to NumPy array
+    seq_array = np.array(sequence, dtype='<U1')
+
+    # Replace invalid bases with 'N'
+    seq_array = np.where(np.isin(seq_array, mapping), seq_array, 'N')
+
+    # Create one-hot encoding matrix
+    one_hot_matrix = (seq_array[:, None] == mapping).astype(int)
+
+    # Flatten and return
+    return one_hot_matrix.flatten()
+
+    # import torch
+    # bases = torch.tensor([ord('A'), ord('C'), ord('G'), ord('T'), ord('N')], dtype=torch.int8, device=device)
+
+    # # Convert input to tensor of character ASCII codes
+    # seq_tensor = torch.tensor([ord(c) for c in sequence], dtype=torch.int8, device=device)
+
+    # # Handle empty sequence
+    # if seq_tensor.numel() == 0:
+    #     print("Warning: Empty sequence encountered in one_hot_encode_torch()")
+    #     return torch.zeros(len(bases), device=device)
+
+    # # Replace invalid bases with 'N'
+    # is_valid = (seq_tensor[:, None] == bases)  # Compare each base with mapping
+    # seq_tensor = torch.where(is_valid.any(dim=1), seq_tensor, ord('N'))
+
+    # # Create one-hot encoding matrix
+    # one_hot_matrix = (seq_tensor[:, None] == bases).int()
+
+    # # Flatten and return
+    # return one_hot_matrix.flatten()

smftools/informatics/helpers/plot_read_length_and_coverage_histograms.py

@@ -0,0 +1,53 @@
+# plot_read_length_and_coverage_histograms
+
+def plot_read_length_and_coverage_histograms(bed_file, plotting_directory):
+    """
+    Plots read length and coverage statistics for each record.
+
+    Parameters:
+        bed_file (str): Path to the bed file to derive read lengths and coverage from.
+        plot_directory (str): Path to the directory to write out historgrams.
+
+    Returns:
+        None
+    """
+    import pandas as pd
+    import matplotlib.pyplot as plt
+    import numpy as np
+    import os
+
+    bed_basename = os.path.basename(bed_file).split('.bed')[0]
+    # Load the BED file into a DataFrame
+    print(f"Loading BED to plot read length and coverage histograms: {bed_file}")
+    df = pd.read_csv(bed_file, sep='\t', header=None, names=['chromosome', 'start', 'end', 'length', 'read_name'])
+    
+    # Group by chromosome
+    grouped = df.groupby('chromosome')
+
+    for chrom, group in grouped:
+        # Plot read length histogram
+        plt.figure(figsize=(12, 6))
+        plt.hist(group['length'], bins=50, edgecolor='k', alpha=0.7)
+        plt.title(f'Read Length Histogram of reads aligned to {chrom}')
+        plt.xlabel('Read Length')
+        plt.ylabel('Count')
+        plt.grid(True)
+        save_name = os.path.join(plotting_directory, f'{bed_basename}_{chrom}_read_length_histogram.png')
+        plt.savefig(save_name)
+        plt.close()
+
+        # Compute coverage
+        coverage = np.zeros(group['end'].max())
+        for _, row in group.iterrows():
+            coverage[row['start']:row['end']] += 1
+        
+        # Plot coverage histogram
+        plt.figure(figsize=(12, 6))
+        plt.plot(coverage, color='b')
+        plt.title(f'Coverage Histogram for {chrom}')
+        plt.xlabel('Position')
+        plt.ylabel('Coverage')
+        plt.grid(True)
+        save_name = os.path.join(plotting_directory, f'{bed_basename}_{chrom}_coverage_histogram.png')
+        plt.savefig(save_name)
+        plt.close()

smftools/informatics/helpers/run_multiqc.py

@@ -0,0 +1,28 @@
+def run_multiqc(input_dir, output_dir):
+    """
+    Runs MultiQC on a given directory and saves the report to the specified output directory.
+
+    Parameters:
+    - input_dir (str): Path to the directory containing QC reports (e.g., FastQC, Samtools, bcftools outputs).
+    - output_dir (str): Path to the directory where MultiQC reports should be saved.
+
+    Returns:
+    - None: The function executes MultiQC and prints the status.
+    """
+    import os
+    import subprocess
+    # Ensure the output directory exists
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Construct MultiQC command
+    command = ["multiqc", input_dir, "-o", output_dir]
+
+    print(f"Running MultiQC on '{input_dir}' and saving results to '{output_dir}'...")
+    
+    # Run MultiQC
+    try:
+        subprocess.run(command, check=True)
+        print(f"MultiQC report generated successfully in: {output_dir}")
+    except subprocess.CalledProcessError as e:
+        print(f"Error running MultiQC: {e}")
+

smftools/informatics/helpers/separate_bam_by_bc.py

@@ -0,0 +1,43 @@
+## separate_bam_by_bc
+
+# General
+def separate_bam_by_bc(input_bam, output_prefix, bam_suffix, split_dir):
+    """
+    Separates an input BAM file on the BC SAM tag values.
+
+    Parameters:
+        input_bam (str): File path to the BAM file to split.
+        output_prefix (str): A prefix to append to the output BAM.
+        bam_suffix (str): A suffix to add to the bam file.
+        split_dir (str): String indicating path to directory to split BAMs into
+    
+    Returns:
+        None
+            Writes out split BAM files.
+    """
+    import pysam
+    import os
+
+    bam_base = os.path.basename(input_bam)
+    bam_base_minus_suffix = bam_base.split(bam_suffix)[0]
+
+    # Open the input BAM file for reading
+    with pysam.AlignmentFile(input_bam, "rb") as bam:
+        # Create a dictionary to store output BAM files
+        output_files = {}
+        # Iterate over each read in the BAM file
+        for read in bam:
+            try:
+                # Get the barcode tag value
+                bc_tag = read.get_tag("BC", with_value_type=True)[0].split('barcode')[1]
+                # Open the output BAM file corresponding to the barcode
+                if bc_tag not in output_files:
+                    output_path = os.path.join(split_dir, f"{output_prefix}_{bam_base_minus_suffix}_{bc_tag}{bam_suffix}")
+                    output_files[bc_tag] = pysam.AlignmentFile(output_path, "wb", header=bam.header)
+                # Write the read to the corresponding output BAM file
+                output_files[bc_tag].write(read)
+            except KeyError:
+                 print(f"BC tag not present for read: {read.query_name}")
+    # Close all output BAM files
+    for output_file in output_files.values():
+        output_file.close()

smftools/informatics/helpers/split_and_index_BAM.py

@@ -0,0 +1,36 @@
+## split_and_index_BAM
+
+def split_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix, output_directory):
+    """
+    A wrapper function for splitting BAMS and indexing them.
+    Parameters:
+        aligned_sorted_BAM (str): A string representing the file path of the aligned_sorted BAM file.
+        split_dir (str): A string representing the file path to the directory to split the BAMs into.
+        bam_suffix (str): A suffix to add to the bam file.
+        output_directory (str): A file path to the directory to output all the analyses.
+    
+    Returns:
+        None
+            Splits an input BAM file on barcode value and makes a BAM index file.
+    """
+    from .. import readwrite
+    import os
+    import subprocess
+    import glob
+    from .separate_bam_by_bc import separate_bam_by_bc
+    from .make_dirs import make_dirs
+
+    plotting_dir = os.path.join(output_directory, 'demultiplexed_bed_histograms')
+    bed_dir = os.path.join(output_directory, 'demultiplexed_read_alignment_coordinates')
+    make_dirs([plotting_dir, bed_dir])
+    aligned_sorted_output = aligned_sorted_BAM + bam_suffix
+    file_prefix = readwrite.date_string()
+    separate_bam_by_bc(aligned_sorted_output, file_prefix, bam_suffix, split_dir)
+    # Make a BAM index file for the BAMs in that directory
+    bam_pattern = '*' + bam_suffix
+    bam_files = glob.glob(os.path.join(split_dir, bam_pattern))
+    bam_files = [bam for bam in bam_files if '.bai' not in bam]
+    for input_file in bam_files:
+        subprocess.run(["samtools", "index", input_file])
+
+    return bam_files

smftools/informatics/load_adata.py

@@ -0,0 +1,182 @@
+## load_adata
+
+def load_adata(config_path):
+    """
+    High-level function to call for converting raw sequencing data to an adata object. 
+    Works for nanopore pod5, fast5, and unaligned modBAM data types for direct SMF workflows.
+    Works for nanopore pod5, fast5, unaligned BAM for conversion SMF workflows.
+    Also works for illumina fastq and unaligned BAM for conversion SMF workflows.
+
+    Parameters:
+        config_path (str): A string representing the file path to the experiment configuration csv file.
+
+    Returns:
+        None
+    """
+    # Lazy importing of packages
+    from .helpers import LoadExperimentConfig, make_dirs, concatenate_fastqs_to_bam, extract_read_features_from_bam
+    from .fast5_to_pod5 import fast5_to_pod5
+    from .subsample_fasta_from_bed import subsample_fasta_from_bed
+    import os
+    import numpy as np
+    import anndata as ad
+    from pathlib import Path
+
+    # Default params
+    bam_suffix = '.bam' # If different, change from here.
+    split_dir = 'demultiplexed_BAMs' # If different, change from here.
+    strands = ['bottom', 'top'] # If different, change from here. Having both listed generally doesn't slow things down too much.
+    conversions = ['unconverted'] # The name to use for the unconverted files. If different, change from here.
+
+    # Load experiment config parameters into global variables
+    experiment_config = LoadExperimentConfig(config_path)
+    var_dict = experiment_config.var_dict
+
+    # These below variables will point to default_value if they are empty in the experiment_config.csv or if the variable is fully omitted from the csv.
+    default_value = None
+
+    # General config variable init
+    smf_modality = var_dict.get('smf_modality', default_value) # needed for specifying if the data is conversion SMF or direct methylation detection SMF. Necessary.
+    input_data_path = var_dict.get('input_data_path', default_value) # Path to a directory of POD5s/FAST5s or to a BAM/FASTQ file. Necessary.
+    output_directory = var_dict.get('output_directory', default_value) # Path to the output directory to make for the analysis. Necessary.
+    fasta = var_dict.get('fasta', default_value) # Path to reference FASTA.
+    fasta_regions_of_interest = var_dict.get("fasta_regions_of_interest", default_value) # Path to a bed file listing coordinate regions of interest within the FASTA to include. Optional.
+    mapping_threshold = var_dict.get('mapping_threshold', default_value) # Minimum proportion of mapped reads that need to fall within a region to include in the final AnnData.
+    experiment_name = var_dict.get('experiment_name', default_value) # A key term to add to the AnnData file name.
+    model_dir = var_dict.get('model_dir', default_value) # needed for dorado basecaller
+    model = var_dict.get('model', default_value) # needed for dorado basecaller
+    barcode_kit = var_dict.get('barcode_kit', default_value) # needed for dorado basecaller
+    barcode_both_ends = var_dict.get('barcode_both_ends', default_value) # dorado demultiplexing
+    trim = var_dict.get('trim', default_value) # dorado adapter and barcode removal
+    input_already_demuxed = var_dict.get('input_already_demuxed', default_value) # If the input files are already demultiplexed.
+    threads = var_dict.get('threads', default_value) # number of cpu threads available for multiprocessing
+    # Conversion specific variable init
+    conversion_types = var_dict.get('conversion_types', default_value)
+    # Direct methylation specific variable init
+    filter_threshold = var_dict.get('filter_threshold', default_value)
+    m6A_threshold = var_dict.get('m6A_threshold', default_value)
+    m5C_threshold = var_dict.get('m5C_threshold', default_value)
+    hm5C_threshold = var_dict.get('hm5C_threshold', default_value)
+    thresholds = [filter_threshold, m6A_threshold, m5C_threshold, hm5C_threshold]
+    mod_list = var_dict.get('mod_list', default_value)
+    batch_size = var_dict.get('batch_size', default_value)
+    device = var_dict.get('device', 'auto')
+    make_bigwigs = var_dict.get('make_bigwigs', default_value)
+    skip_unclassified = var_dict.get('skip_unclassified', True)
+    delete_batch_hdfs = var_dict.get('delete_batch_hdfs', True)
+
+    # Make initial output directory
+    make_dirs([output_directory])
+    os.chdir(output_directory)
+    # Define the pathname to split BAMs into later during demultiplexing.
+    split_path = os.path.join(output_directory, split_dir)
+
+    # If fasta_regions_of_interest is passed, subsample the input FASTA on regions of interest and use the subsampled FASTA.
+    if fasta_regions_of_interest and '.bed' in fasta_regions_of_interest:
+        fasta_basename = os.path.basename(fasta).split('.fa')[0]
+        bed_basename_minus_suffix = os.path.basename(fasta_regions_of_interest).split('.bed')[0]
+        output_FASTA = fasta_basename + '_subsampled_by_' + bed_basename_minus_suffix + '.fasta'
+        subsample_fasta_from_bed(fasta, fasta_regions_of_interest, output_directory, output_FASTA)
+        fasta = os.path.join(output_directory, output_FASTA)
+
+    # If conversion_types is passed:
+    if conversion_types:
+        conversions += conversion_types
+
+    # Get the input filetype
+    if Path(input_data_path).is_file():
+        input_data_filetype = '.' + os.path.basename(input_data_path).split('.')[1].lower()
+        input_is_pod5 = input_data_filetype in ['.pod5','.p5']
+        input_is_fast5 = input_data_filetype in ['.fast5','.f5']
+        input_is_fastq = input_data_filetype in ['.fastq', '.fq']
+        input_is_bam = input_data_filetype == bam_suffix
+        if input_is_fastq:
+            fastq_paths = [input_data_path]
+    elif Path(input_data_path).is_dir():
+        # Get the file names in the input data dir
+        input_files = os.listdir(input_data_path)
+        input_is_pod5 = sum([True for file in input_files if '.pod5' in file or '.p5' in file])
+        input_is_fast5 = sum([True for file in input_files if '.fast5' in file or '.f5' in file])
+        input_is_fastq = sum([True for file in input_files if '.fastq' in file or '.fq' in file])
+        input_is_bam = sum([True for file in input_files if bam_suffix in file])
+        if input_is_fastq:
+            fastq_paths = [os.path.join(input_data_path, file) for file in input_files if '.fastq' in file or '.fq' in file]
+
+    # If the input files are not pod5 files, and they are fast5 files, convert the files to a pod5 file before proceeding.
+    if input_is_fast5 and not input_is_pod5:
+        # take the input directory of fast5 files and write out a single pod5 file into the output directory.
+        output_pod5 = os.path.join(output_directory, 'FAST5s_to_POD5.pod5')
+        print(f'Input directory contains fast5 files, converting them and concatenating into a single pod5 file in the {output_pod5}')
+        fast5_to_pod5(input_data_path, output_pod5)
+        # Reassign the pod5_dir variable to point to the new pod5 file.
+        input_data_path = output_pod5
+        input_is_pod5 = True
+        input_is_fast5 = False
+    
+    elif input_is_fastq:
+        output_bam = os.path.join(output_directory, 'FASTQs_concatenated_into_BAM.bam')
+        concatenate_fastqs_to_bam(fastq_paths, output_bam, barcode_tag='BC', gzip_suffix='.gz')
+        input_data_path = output_bam
+        input_is_bam = True
+        input_is_fastq = False
+
+    if input_is_pod5:
+        basecall = True
+    elif input_is_bam:
+        basecall = False
+    else:
+        print('Error, can not find input bam or pod5')
+
+    if smf_modality == 'conversion':
+        from .conversion_smf import conversion_smf
+        final_adata, final_adata_path, sorted_output, bam_files = conversion_smf(fasta, output_directory, conversions, strands, model_dir, model, input_data_path, split_path
+                                                         , barcode_kit, mapping_threshold, experiment_name, bam_suffix, basecall, barcode_both_ends, trim, device, make_bigwigs, threads, input_already_demuxed)
+    elif smf_modality == 'direct':
+        from .direct_smf import direct_smf
+        # need to add input_already_demuxed workflow here.
+        final_adata, final_adata_path, sorted_output, bam_files = direct_smf(fasta, output_directory, mod_list,model_dir, model, thresholds, input_data_path, split_path
+                                                     , barcode_kit, mapping_threshold, experiment_name, bam_suffix, batch_size, basecall, barcode_both_ends, trim, device, make_bigwigs, skip_unclassified, delete_batch_hdfs, threads)
+    else:
+            print("Error")
+            
+    # Read in the final adata object and append final metadata
+    #print(f'Reading in adata from {final_adata_path} to add final metadata')
+    # final_adata = ad.read_h5ad(final_adata_path)
+    
+    # Adding read query length metadata to adata object.
+    read_metrics = {}
+    for bam_file in bam_files:
+        bam_read_metrics = extract_read_features_from_bam(bam_file)
+        read_metrics.update(bam_read_metrics)
+    #read_metrics = extract_read_features_from_bam(sorted_output)
+
+    query_read_length_values = []
+    query_read_quality_values = []
+    reference_lengths = []
+    # Iterate over each row of the AnnData object
+    for obs_name in final_adata.obs_names:
+        # Fetch the value from the dictionary using the obs_name as the key
+        value = read_metrics.get(obs_name, np.nan)  # Use np.nan if the key is not found
+        if type(value) is list:
+            query_read_length_values.append(value[0])
+            query_read_quality_values.append(value[1])
+            reference_lengths.append(value[2])
+        else:
+            query_read_length_values.append(value)
+            query_read_quality_values.append(value)
+            reference_lengths.append(value) 
+                       
+    # Add the new column to adata.obs
+    final_adata.obs['query_read_length'] = query_read_length_values
+    final_adata.obs['query_read_quality'] = query_read_quality_values
+    final_adata.obs['query_length_to_reference_length_ratio'] = np.array(query_read_length_values) / np.array(reference_lengths)
+
+    final_adata.obs['Raw_methylation_signal'] = np.nansum(final_adata.X, axis=1)
+    final_adata.obs['Raw_per_base_methylation_average'] = final_adata.obs['Raw_methylation_signal'] / final_adata.obs['query_read_length']
+
+    print('Saving final adata')
+    if ".gz" in final_adata_path:
+        final_adata.write_h5ad(f"{final_adata_path}", compression='gzip')
+    else:
+        final_adata.write_h5ad(f"{final_adata_path}.gz", compression='gzip')
+    print('Final adata saved')

smftools/informatics/readwrite.py

@@ -0,0 +1,106 @@
+## readwrite ##
+
+######################################################################################################
+## Datetime functionality
+def date_string():
+    """
+    Each time this is called, it returns the current date string
+    """
+    from datetime import datetime
+    current_date = datetime.now()
+    date_string = current_date.strftime("%Y%m%d")
+    date_string = date_string[2:]
+    return date_string
+
+def time_string():
+    """
+    Each time this is called, it returns the current time string
+    """
+    from datetime import datetime
+    current_time = datetime.now()
+    return current_time.strftime("%H:%M:%S")
+######################################################################################################
+
+######################################################################################################
+## Numpy, Pandas, Anndata functionality
+def adata_to_df(adata, layer=None):
+    """
+    Input: An adata object with a specified layer.
+    Output: A dataframe for the specific layer.
+    """
+    import pandas as pd
+    import anndata as ad
+
+    # Extract the data matrix from the given layer
+    if layer:
+        data_matrix = adata.layers[layer]
+    else:
+        data_matrix = adata.X
+    # Extract observation (read) annotations
+    obs_df = adata.obs
+    # Extract variable (position) annotations
+    var_df = adata.var
+    # Convert data matrix and annotations to pandas DataFrames
+    df = pd.DataFrame(data_matrix, index=obs_df.index, columns=var_df.index)
+    return df
+
+def save_matrix(matrix, save_name):
+    """
+    Input: A numpy matrix and a save_name
+    Output: A txt file representation of the data matrix
+    """
+    import numpy as np
+    np.savetxt(f'{save_name}.txt', matrix)
+
+def concatenate_h5ads(output_file, file_suffix='h5ad.gz', delete_inputs=True):
+    """
+    Concatenate all h5ad files in a directory and delete them after the final adata is written out.
+    Input: an output file path relative to the directory in which the function is called
+    """
+    import os
+    import anndata as ad
+    # Runtime warnings
+    import warnings
+    warnings.filterwarnings('ignore', category=UserWarning, module='anndata')
+    warnings.filterwarnings('ignore', category=FutureWarning, module='anndata')
+    
+    # List all files in the directory
+    files = os.listdir(os.getcwd())
+    # get current working directory
+    cwd = os.getcwd()
+    suffix = file_suffix
+    # Filter file names that contain the search string in their filename and keep them in a list
+    hdfs = [hdf for hdf in files if suffix in hdf]
+    # Sort file list by names and print the list of file names
+    hdfs.sort()
+    print('{0} sample files found: {1}'.format(len(hdfs), hdfs))
+    # Iterate over all of the hdf5 files and concatenate them.
+    final_adata = None
+    for hdf in hdfs:
+        print('{0}: Reading in {1} hdf5 file'.format(time_string(), hdf))
+        temp_adata = ad.read_h5ad(hdf)
+        if final_adata:
+            print('{0}: Concatenating final adata object with {1} hdf5 file'.format(time_string(), hdf))
+            final_adata = ad.concat([final_adata, temp_adata], join='outer', index_unique=None)
+        else:
+            print('{0}: Initializing final adata object with {1} hdf5 file'.format(time_string(), hdf))
+            final_adata = temp_adata
+    print('{0}: Writing final concatenated hdf5 file'.format(time_string()))
+    final_adata.write_h5ad(output_file, compression='gzip')
+
+    # Delete the individual h5ad files and only keep the final concatenated file
+    if delete_inputs:
+        files = os.listdir(os.getcwd())
+        hdfs = [hdf for hdf in files if suffix in hdf]
+        if output_file in hdfs:
+            hdfs.remove(output_file)
+            # Iterate over the files and delete them
+            for hdf in hdfs:
+                try:
+                    os.remove(hdf)
+                    print(f"Deleted file: {hdf}")
+                except OSError as e:
+                    print(f"Error deleting file {hdf}: {e}")
+    else:
+        print('Keeping input files')
+######################################################################################################