XspecT 0.1.3__py3-none-any.whl → 0.2.0__py3-none-any.whl

xspect/train_filter/README_XspecT_Erweiterung.md

@@ -1,119 +0,0 @@
-# XspecT-Erweiterung
-
-Expands XspecT, so new filter for a genus can automatically be trained. It's main 
-script is XspecT_trainer.py. The rest of the scripts are inside the python module 
-train_filter. 
-
-## Training new filter
-
-XspecT_trainer.py uses command line arguments. The examples for using XspecT_trainer.py
-are using Salmonella since this genus only has two defined species in the NCBI 
-databases.
-
-### Jellyfish
-
-The program jellyfish is used to count distinct k-meres in the assemblies. For XspecT_
-trainer.py to work jellyfish needs to be installed. It can be installed using bioconda:
-
-`
-conda install -c bioconda jellyfish
-`
-
-### Training examples
-
-New filters with assemblies from NCBI RefSeq can be trained with the following line. The 
-python libraries from [requirements.txt](..%2Frequirements.txt) need to be installed.
-
-`
-python XspecT_trainer.py Salmonella 1
-`
-
-Training filters with custom data can be done using the following line.
-
-`
-python XspecT_trainer.py Salmonella 2 -bf /path/to/concate_assemblies -svm 
-/path/to/assemblies
-`
-
-All command line arguments are explained using the following line.
-
-`
-python XspecT_trainer.py -h
-`
-
-# Explanation of the scripts
-
-## backup_filter.py
-
-Creates a backup of all files needed for the species assignment by XspecT for a specific
-genus. The backup will be done, if new filters will be created for a genus which 
-already has trained filters.
-
-## create_svm.py
-
-Downloads the needed assemblies and trains a support-vector-machine for the genus.
-
-## extract_and_concatenate.py
-
-Unzips the downloaded assemblies. Concatenates assemblies per species that will be used 
-to train the bloomfilters.
-
-## get_paths.py
-
-Functions that get specific paths.
-
-## html_scrap.py
-
-Updates a list of all NCBI RefSeq assembly accessions that have a taxonomy check result
-of OK. The taxonomy check from NCBI RefSeq uses the ANI (average-nucleotide-
-identity) to compute a result.
-
-## interface_XspecT.py
-
-Mostly functions that train new bloomfilters automatically. The functions were 
-originally writen for XspecT in a non-automatic way and were updated.
-
-## k_mer_count.py
-
-Uses jellyfish to count distinct k-meres in every concatenated assembly. The highest
-count will be used to compute the size of the bloomfilters.
-
-## ncbi_api
-
-A module which makes requests to the NCBI Datasets API.
-
-### download_assemblies.py
-
-The specific function that downloads assemblies from NCBI RefSeq using NCBI 
-datasets.
-
-### ncbi_assembly_metadata.py
-
-Takes a dictionary with species and their taxon ID and asks NCBI for assemblies of
-the species. Saves the collected accessions of the found and selected assemblies.
-
-### ncbi_children_tree.py
-
-Takes the name or ID of a genus and gives a list with all its species.
-
-### ncbi_taxon_metadata.py
-
-Takes a list with taxon and collects metadata like their scientific name and rank.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

xspect/train_filter/get_paths.py

@@ -1,35 +0,0 @@
-from pathlib import Path
-import os
-
-
-def get_concatenate_file_path(dir_name):
-    """Returns str to file path of the concatenate directory.
-
-    :param dir_name: Name of the current genus_metadata directory.
-    :type dir_name: str
-    :return: File path to the concatenated species assemblies.
-    """
-    return Path(os.getcwd()) / "genus_metadata" / dir_name / "concatenate"
-
-
-def get_current_dir_file_path(dir_name):
-    """Returns str of file path to the directory with the currently needed metagenome assembly.
-
-    :param dir_name: Name of the current genus_metadata directory.
-    :type dir_name: str
-    :return: File path to the metagenome assembly.
-    """
-    return Path(os.getcwd()) / "genus_metadata" / dir_name
-
-
-def get_metagenome_filter_path():
-    """Returns the file path to the metagenome filters."""
-    return Path(os.getcwd()) / "filter" / "Metagenomes"
-
-
-def main():
-    pass
-
-
-if __name__ == "__main__":
-    main()

xspect/train_filter/interface_XspecT.py

@@ -1,204 +0,0 @@
-import os
-import pickle
-from pathlib import Path
-from shutil import rmtree
-
-from loguru import logger
-from numpy import log, square
-
-import xspect.BF_v2 as BF_v2
-
-
-def compute_array_size(n, p=0.01):
-    """Computes the Bit-Array-Size for the bloomfilters.
-
-    :param n: Highest k-mer count of a species.
-    :type n: int
-    :param p: Rate of mistakes.
-    :type p: float
-    :return: Bit-Array-Size for the bloomfilters.
-    """
-    return -((n * log(p)) / (square(log(2))))
-
-
-def make_paths(dir_name, genus):
-    """Create paths to the concatenated sequences and to where the new bloomfilters will be saved.
-
-    :param dir_name: Name of the parent directory.
-    :type dir_name: str
-    :param genus: Name of the genus.
-    :type genus: str
-    :return: The path to the sequence files and the bloomfilter directory.
-    """
-    # Path to concatenated sequences
-    files_path = Path(os.getcwd()) / "genus_metadata" / dir_name / "concatenate"
-
-    # Path for results.
-    result_path = Path(os.getcwd()) / "filter" / genus
-    # Try to create the directory for the bloomfilters.
-    try:
-        os.mkdir(result_path)
-    except FileExistsError:
-        # Delete the old directory with bloomfilters if already existed.
-        rmtree(result_path, ignore_errors=False, onerror=None)
-        os.mkdir(result_path)
-
-    return str(files_path), str(result_path)
-
-
-def init_bf(array_size, clonetypes=1, hashes=7, k=21):
-    """Initiates an bloomfilter object  with given parameters.
-
-    :param array_size: The size for the byte-array.
-    :type array_size: int
-    :param clonetypes: Number of clonetypes.
-    :type clonetypes: int
-    :param hashes: Number of hash functions used.
-    :type hashes: int
-    :param k: Length of k-mers.
-    :type k: int
-    :return: The initiated bloomfilter object.
-    """
-    BF = BF_v2.AbaumanniiBloomfilter(array_size)
-    BF.set_arraysize(array_size)
-    BF.set_clonetypes(clonetypes)
-    BF.set_hashes(hashes)
-    BF.set_k(k)
-    return BF
-
-
-def new_train_core(files_path, result_path, array_size, k=21):
-    """Trains concatenated genomes into Bloomfilter and saves them.
-
-    :param files_path: Path to where the concatenated sequences are stored.
-    :type files_path: str
-    :param result_path: Path where the generated Bloomfilter will be saved.
-    :type result_path: str
-    :param array_size: Array-size for the Bloomfilter.
-    :type array_size: int
-    :param k: Length of substring.
-    :type k: int
-    """
-    files = os.listdir(files_path)
-    # Iterate the files backwards to delete all non fasta files from the list.
-    for i in range(len(files) - 1, -1, -1):
-        if "fna" in files[i] or "fasta" in files[i]:
-            continue
-        else:
-            del files[i]
-
-    # Train a bloomfilter for each species.
-    for i in range(len(files)):
-        BF = init_bf(array_size=array_size, clonetypes=1, hashes=7, k=k)
-        path = Path(files_path) / files[i]
-        species_name = files[i].split(".")[0]
-        file_name = species_name + ".txt"
-        logger.info("Training {name}", name=species_name)
-        result = Path(result_path) / file_name
-        BF.train_sequence(path, 0)
-        BF.save_clonetypes(result)
-        BF.cleanup()
-
-
-def new_write_file_dyn(bf_path, genus, meta_mode=False):
-    """Write file with pickled list of all names for the bloomfilters.
-
-    :param bf_path: Path to the bloomfilters.
-    :type bf_path: str
-    :param genus: Name of the genus.
-    :type genus: str
-    :param meta_mode: Declare to which bloomfilters the path leads.
-    :type meta_mode: bool
-    """
-    files = os.listdir(bf_path)
-    # If the Bloomfilter path leads to Bloomfilter for the metagenome mode.
-    if meta_mode:
-        for i in range(len(files) - 1, -1, -1):
-            if genus not in files[i]:
-                del files[i]
-            else:
-                files[i] = files[i][:-4]
-        file_name = "Filter" + genus + "Complete.txt"
-
-    # If the path leads to bloomfilters for the species.
-    else:
-        for i in range(len(files) - 1, -1, -1):
-            if "txt" not in files[i]:
-                del files[i]
-            else:
-                files[i] = files[i][:-4]
-        file_name = "Filter" + genus + ".txt"
-
-    # Make path for the txt file.
-    file_path = Path(os.getcwd()) / "filter" / "species_names" / file_name
-    with open(file_path, "wb") as fp:
-        pickle.dump(sorted(files), fp)
-
-
-def save_array_sizes(genus, array_sizes):
-    """Saves the array sizes of the bytearray for the bloomfilters in a txt file.
-
-    :param genus: The current genus.
-    :type genus: str
-    :param array_sizes: List of all computed array sizes for this genus.
-    :type array_sizes: list[str]
-    """
-    file_name = genus + ".txt"
-    path = Path(os.getcwd()) / "filter" / "array_sizes" / file_name
-
-    # Save both array sizes as a string in the format: 'size1 size2' as a txt file.
-    # The first size is of the species level filters and the second of the meta-mode filter.
-    text = " ".join(array_sizes)
-    with open(path, "w", encoding="utf-8") as f:
-        f.write(text)
-
-
-def save_name_dict(genus, name_dict: dict):
-    """Saves the names and taxon IDs of all species for which filter were trained. XspecT uses this dict to switch
-    between the species names and it's ID. The dict is saved as a csv file.
-
-    :param genus: The genus for which filters were trained.
-    :param name_dict: The dictionary with all species names and taxon IDs
-    """
-
-
-def save_time_stats(time_stats, dir_name):
-    """Saves the collected time measurements as a txt file.
-
-    :param time_stats: The collected time measurements as a formatted string.
-    :type time_stats: str
-    :param dir_name: Name of the parent directory.
-    :type dir_name: str
-    """
-    time_file = Path(os.getcwd()) / "genus_metadata" / dir_name / "time.txt"
-    with open(str(time_file), "w+", encoding="utf-8") as f:
-        f.write(time_stats)
-
-
-def load_translation_dict(genus: str) -> dict[str, str]:
-    """Loads the translation dict for the given genus. The key is the taxon ID and its value the scientific name.
-
-    :param genus: The name of the genus.
-    :return: The translation dict for the genus.
-    """
-    file_name = f"{genus}.pickle"
-    path = Path(os.getcwd()) / "filter" / "translation_dicts" / file_name
-    with open(path, "rb") as f:
-        translation_dict = pickle.load(f)
-
-    return translation_dict
-
-
-def main():
-    a = 28858023
-    b = compute_array_size(a)
-    print(int(round(b + 1000000, -6)))
-    # genera = get_genera_array_sizes()
-    # print(f"Species: ")
-    # print(pre_process_all(genera, meta_mode=False))
-    # print(f"\nMeta: ")
-    # print(pre_process_all(genera, meta_mode=True))
-
-
-if __name__ == "__main__":
-    main()

xspect/train_filter/k_mer_count.py

@@ -1,162 +0,0 @@
-import subprocess as sp
-from linecache import getline
-from os import listdir, remove, getcwd
-from pathlib import Path
-from time import perf_counter, localtime, asctime
-
-from loguru import logger
-import numpy as np
-
-import xspect.XspecT_trainer
-
-
-def get_seq_paths(dir_name: str):
-    """Stores the sequence paths, with the species name as key, in a dictionary. The sequences are DNA-Assemblies which
-    were concatenated.
-
-    :param dir_name: Name of the directory.
-    :return: Dictionary with species names and sequences.
-    """
-    dir_path = Path(getcwd()) / "genus_metadata" / dir_name / "concatenate"
-    sequence_dict = dict()
-    files = listdir(dir_path)
-    # Go through all files backwards to delete all non fasta files.
-    for i in range(len(files) - 1, -1, -1):
-        curr_file = str(files[i])
-        file_parts = curr_file.split(".")
-        if file_parts[-1] != "fasta":
-            del files[i]
-        else:
-            species_name = file_parts[0]
-            # Save the species name with the path to its fasta file.
-            sequence_dict[species_name] = str(dir_path / curr_file)
-
-    return sequence_dict
-
-
-def jellyfish_count(command: str):
-    """A jellyfish command to count the k-mers of an fasta file using the linux bash.
-
-    :param command: The jellyfish command with all chosen parameters.
-    """
-    sp.run(command.split(" "))
-
-
-def jellyfish_stats(command: str) -> int:
-    """A jellyfish command to get the count which was
-
-    :param command: The jellyfish command with all chosen parameters
-    :return: The count of all distinct k-mers.
-    """
-    result = sp.run(command.split(" "), stdout=sp.PIPE, text=True)
-    return int(result.stdout.split("\n")[1].replace(" ", "").split(":")[1])
-
-
-def count_k_meres(sequence_dict, k=21):
-    """Counts all k-meres in the sequences using jellyfish.
-
-    :param sequence_dict: Dictionary with all sequence paths.
-    :type sequence_dict: dict[str, str]
-    :param k: K-mer length.
-    :type k: int
-    :return: Species names and number of distinct k-mere.
-    """
-    k_mer_of_species = list()
-    count = 0
-
-    # Iterate through all species.
-    for species_name, file_path in sequence_dict.items():
-        num_files_to_count = len(sequence_dict) - count
-        logger.info(
-            "{num} files left to count. Counting {name}",
-            num=num_files_to_count,
-            name=species_name,
-        )
-        count += 1
-
-        # Set parameters for jellyfish commands.
-        k = str(k)
-        hash_size = "100M"
-        num_threads = "4"
-        output_name = str(Path(getcwd()) / "output")
-
-        # Command for jellyfish count.
-        count_command = (
-            "jellyfish count -m "
-            + k
-            + " -o "
-            + output_name
-            + " -C -s "
-            + hash_size
-            + " -t "
-            + num_threads
-            + " "
-            + file_path
-        )
-        # Command for jellyfish stats.
-        stats_command = "jellyfish stats " + output_name
-        jellyfish_count(count_command)
-        k_mer_count = jellyfish_stats(stats_command)
-
-        # Append tuple with species name and distinct k-mer count.
-        k_mer_of_species.append((species_name, k_mer_count))
-
-    return k_mer_of_species
-
-
-def sort_k_mer_counts(k_mer_counts):
-    """Sorts the list of k-mers to determine the highest count.
-
-    :param k_mer_counts: List of all species with their k-mer counts.
-    :type k_mer_counts: list[tuple[str, int]]
-    :return: Sorted list beginning with the highest k-mer count.
-    """
-    # Define the data type for numpy.
-    data_type = [("species", "S50"), ("k_mer_count", int)]
-    # Create numpy array with defined data type.
-    k_mer_count_sorted = np.array(k_mer_counts, dtype=data_type)
-    # Sort array based on k-mer count and than reverse so the first tuple has the highest count.
-    k_mer_count_sorted = np.sort(k_mer_count_sorted, order="k_mer_count")[::-1]
-
-    return k_mer_count_sorted
-
-
-def get_highest_k_mer_count(dir_name, k=21):
-    """Gets highest k-mer count for all species and k-mer count of genus.
-
-    :param dir_name: Name of the parent directory.
-    :type dir_name: str
-    :param k: K-mer length.
-    :type k: int
-    :return: List of the highest k-mer count of all species and the k-mer count of all sequences united.
-    """
-    # Get highest k-mer count of all species.
-    seq_dict = get_seq_paths(dir_name)
-    k_mer_counts = count_k_meres(seq_dict, k=k)
-    k_mer_sorted = sort_k_mer_counts(k_mer_counts)
-    # Uncomment if the k-mer counts should be saved.
-    # save_count(k_mer_sorted, dir_name)
-
-    # Count distinct k-mers of genus.
-    genus = dir_name.split("_")[0]
-    file_name = genus + ".fasta"
-    file_path = str(Path(getcwd()) / "genus_metadata" / dir_name / file_name)
-    seq_dict = {genus: file_path}
-    k_mer_count = count_k_meres(seq_dict, k=k)
-
-    # Return highest k-mer count of all species and k-mer count of complete genus.
-    return [k_mer_sorted[0][1], k_mer_count[0][1]]
-
-
-def main():
-    dir_name = "Listeria_14_12_2022_21-5-13"
-    seq_dict = get_seq_paths(dir_name)
-
-    start = perf_counter()
-
-    end = perf_counter()
-    print(f"time: {(end-start)/60}")
-
-
-if __name__ == "__main__":
-    main()