XspecT 0.2.7__py3-none-any.whl → 0.4.0__py3-none-any.whl

xspect/pipeline.py

@@ -1,201 +0,0 @@
-"""Module for defining the Pipeline class."""
-
-import json
-from pathlib import Path
-from Bio.SeqRecord import SeqRecord
-from Bio import SeqIO
-from xspect.file_io import get_records_by_id
-from xspect.models.result import StepType, SubprocessingStep
-from xspect.run import Run
-from xspect.models.result import ModelResult
-from xspect.model_management import get_model_by_slug
-
-
-class ModelExecution:
-    """Class for storing a processing step of an XspecT pipeline."""
-
-    def __init__(
-        self,
-        model_slug: str,
-        sparse_sampling_step: int = 1,
-    ):
-        self.model_slug = model_slug
-        self.sparse_sampling_step = sparse_sampling_step
-        self.pipeline_steps = []
-
-    def add_pipeline_step(
-        self,
-        pipeline_step: "PipelineStep",
-    ):
-        """Add a subprocessing step to the pipeline step."""
-        self.pipeline_steps.append(pipeline_step)
-
-    def to_dict(self) -> dict:
-        """Return the processing step as a dictionary."""
-        return {
-            "model_slug": self.model_slug,
-            "sparse_sampling_step": self.sparse_sampling_step,
-            "pipeline_steps": [
-                pipeline_step.to_dict() for pipeline_step in self.pipeline_steps
-            ],
-        }
-
-    def run(
-        self,
-        sequence_input: (
-            SeqRecord
-            | list[SeqRecord]
-            | SeqIO.FastaIO.FastaIterator
-            | SeqIO.QualityIO.FastqPhredIterator
-            | Path
-        ),
-    ) -> ModelResult:
-        """Run the model on a given input."""
-        model = get_model_by_slug(self.model_slug)
-        model_result = model.predict(sequence_input, step=self.sparse_sampling_step)
-
-        for pipeline_step in self.pipeline_steps:
-            if pipeline_step.subprocessing_type == StepType.PREDICTION:
-                score = model_result.get_scores()["total"][pipeline_step.label]
-                if score >= pipeline_step.treshold:
-                    prediction_model_result = pipeline_step.model_execution.run(
-                        sequence_input
-                    )
-                    subprocessing_step = SubprocessingStep(
-                        pipeline_step.subprocessing_type,
-                        pipeline_step.label,
-                        pipeline_step.treshold,
-                        prediction_model_result,
-                    )
-                    model_result.add_subprocessing_step(subprocessing_step)
-            elif pipeline_step.subprocessing_type == StepType.FILTERING:
-                filtered_sequence_ids = model_result.get_filtered_subsequences(
-                    pipeline_step.label, pipeline_step.treshold
-                )
-                sequence_input = get_records_by_id(
-                    sequence_input, filtered_sequence_ids
-                )
-
-                filtering_model_result = None
-                if sequence_input:
-                    filtering_model_result = pipeline_step.model_execution.run(
-                        sequence_input
-                    )
-
-                subprocessing_step = SubprocessingStep(
-                    pipeline_step.subprocessing_type,
-                    pipeline_step.label,
-                    pipeline_step.treshold,
-                    filtering_model_result,
-                )
-                model_result.add_subprocessing_step(subprocessing_step)
-            else:
-                raise ValueError(
-                    f"Invalid subprocessing type {pipeline_step.subprocessing_type}"
-                )
-
-        return model_result
-
-
-class PipelineStep:
-    """Class for storing a subprocessing step of an XspecT model."""
-
-    def __init__(
-        self,
-        subprocessing_type: StepType,
-        label: str,
-        treshold: float,
-        model_execution: ModelExecution,
-    ):
-        self.subprocessing_type = subprocessing_type
-        self.label = label
-        self.treshold = treshold
-        self.model_execution = model_execution
-
-    def to_dict(self) -> dict:
-        """Return the subprocessing step as a dictionary."""
-        return {
-            "subprocessing_type": str(self.subprocessing_type),
-            "label": self.label,
-            "treshold": self.treshold,
-            "model_execution": self.model_execution.to_dict(),
-        }
-
-
-class Pipeline:
-    """Class for storing an XspecT pipeline consisting of multiple model processing steps."""
-
-    def __init__(self, display_name: str, author: str, author_email: str):
-        self.display_name = display_name
-        self.author = author
-        self.author_email = author_email
-        self.model_executions = []
-
-    def add_pipeline_step(
-        self,
-        pipeline_step: ModelExecution,
-    ):
-        """Add a processing step to the pipeline."""
-        self.model_executions.append(pipeline_step)
-
-    def to_dict(self) -> dict:
-        """Return the pipeline as a dictionary."""
-        return {
-            "display_name": self.display_name,
-            "author": self.author,
-            "author_email": self.author_email,
-            "model_executions": [
-                model_execution.to_dict() for model_execution in self.model_executions
-            ],
-        }
-
-    def to_json(self) -> str:
-        """Return the pipeline as a JSON string."""
-        return json.dumps(self.to_dict())
-
-    def save(self, path: Path) -> None:
-        """Save the pipeline as a JSON file."""
-        with open(path, "w", encoding="utf-8") as f:
-            f.write(self.to_json())
-
-    @staticmethod
-    def from_file(path: Path) -> "Pipeline":
-        """Load the pipeline from a JSON file."""
-        with open(path, "r", encoding="utf-8") as f:
-            pipeline_json = json.load(f)
-            pipeline = Pipeline(
-                pipeline_json["display_name"],
-                pipeline_json["author"],
-                pipeline_json["author_email"],
-            )
-            for model_execution in pipeline_json["model_executions"]:
-                model_execution = ModelExecution(
-                    model_execution["model_slug"],
-                    model_execution["sparse_sampling_step"],
-                )
-                for pipeline_step in model_execution["pipeline_steps"]:
-                    model_execution.add_pipeline_step(
-                        PipelineStep(
-                            StepType(pipeline_step["subprocessing_type"]),
-                            pipeline_step["label"],
-                            pipeline_step["treshold"],
-                            ModelExecution(
-                                pipeline_step["model_execution"]["model_slug"],
-                                pipeline_step["model_execution"][
-                                    "sparse_sampling_step"
-                                ],
-                            ),
-                        )
-                    )
-                pipeline.add_pipeline_step(model_execution)
-            return pipeline
-
-    def run(self, input_file: Path) -> Run:
-        """Run the pipeline on a given input."""
-        run = Run(self.display_name, input_file)
-
-        for model_execution in self.model_executions:
-            result = model_execution.run(input_file)
-            run.add_result(result)
-
-        return run

xspect/run.py

@@ -1,38 +0,0 @@
-"""Module with XspecT global run class, which summarizes individual model results."""
-
-import json
-from pathlib import Path
-from xspect.models.result import ModelResult
-
-
-class Run:
-    """Class for storing the results of an XspecT run."""
-
-    def __init__(self, display_name: str, input_file: str):
-        self.display_name = display_name
-        self.input_file = input_file
-        self.results = []
-
-    def add_result(self, result: ModelResult):
-        """Add a result to the run."""
-        self.results.append(result)
-
-    def to_dict(self) -> dict:
-        """Return the run as a dictionary."""
-        return {
-            "display_name": self.display_name,
-            "input_file": str(self.input_file),
-            "results": (
-                [result.to_dict() for result in self.results] if self.results else []
-            ),
-        }
-
-    def to_json(self) -> str:
-        """Return the run as a JSON string."""
-        json_dict = self.to_dict()
-        return json.dumps(json_dict, indent=4)
-
-    def save(self, path: Path) -> None:
-        """Save the run as a JSON file."""
-        with open(path, "w", encoding="utf-8") as f:
-            f.write(self.to_json())

xspect/train_filter/create_svm.py

@@ -1,45 +0,0 @@
-"""This module contains functions to select and download assemblies for SVM creation."""
-
-from time import sleep
-from loguru import logger
-from xspect.train_filter.ncbi_api import download_assemblies
-
-
-def select_assemblies(accessions):
-    """Selects up to 4 assemblies, ideally assemblies that were not used for training the filters.
-
-    :param accessions: All selected assembly accessions for every species.
-    :type accessions: dict
-    :return: Dict with species name as key and selected accessions as value.
-    """
-
-    all_accessions = {
-        sci_name: curr_accessions[-4:]
-        for sci_name, curr_accessions in accessions.items()
-    }
-
-    return all_accessions
-
-
-def get_svm_assemblies(all_accessions, dir_name):
-    """Download assemblies for svm creation.
-
-    :param all_accessions: Contains lists with all previously selected assemblies for every species.
-    :type all_accessions: dict
-    :param dir_name: Name of the parent directory.
-    :type dir_name: str
-    """
-    # Select accessions for download.
-    selected_accessions = select_assemblies(all_accessions)
-
-    # Download assemblies.
-    for sci_name, accessions in selected_accessions.items():
-        sleep(5)
-        logger.info("Downloading {name}", name=sci_name)
-        file_name = sci_name + ".zip"
-        download_assemblies.download_assemblies(
-            accessions=accessions,
-            dir_name=dir_name,
-            target_folder="training_data_zipped",
-            zip_file_name=file_name,
-        )

xspect/train_filter/extract_and_concatenate.py

@@ -1,124 +0,0 @@
-"""Module for extracting and concatenating assemblies."""
-
-__author__ = "Berger, Phillip"
-
-import os
-import shutil
-from pathlib import Path
-from Bio import SeqIO
-from xspect import file_io
-from xspect.definitions import get_xspect_tmp_path, fasta_endings
-
-
-def change_header(assemblies_path, species_accessions: dict):
-    """Change the header of the assemblies to the species name."""
-    files = os.listdir(assemblies_path)
-    # Iterate through all species.
-    for name, accessions in species_accessions.items():
-        # Iterate through all accessions of the current species.
-        for accession in accessions:
-            # Iterate through all file names.
-            for file in files:
-                if accession in file:
-                    file_path = assemblies_path / str(file)
-                    # Change the header.
-                    with open(file_path, "r", encoding="utf-8") as f:
-                        sequence = ""
-                        for line in f.readlines():
-                            if line[0] != ">":
-                                sequence += line
-                        new_header = f">{name}\n"
-                    with open(file_path, "w", encoding="utf-8") as f:
-                        f.write(new_header)
-                        f.write(sequence)
-
-
-def copy_assemblies(unzipped_path, assemblies_path):
-    """Copy all assemblies to a new directory."""
-    os.mkdir(assemblies_path)
-    for folder in os.listdir(unzipped_path):
-        for root, _, files in os.walk(unzipped_path / str(folder)):
-            for file in files:
-                file_ending = file.split(".")[-1]
-                if file_ending in fasta_endings:
-                    file_path = Path(root) / file
-                    shutil.copy(file_path, (assemblies_path / file))
-
-
-def concatenate_bf(unzipped_path, concatenate_path):
-    """Concatenate all assemblies for Bloom filter training."""
-
-    all_assemblies = []
-
-    # Make new directory
-    os.mkdir(concatenate_path)
-
-    # Open the fasta files for each species.
-    for folder in os.listdir(unzipped_path):
-        species_files = []
-        # Walk through dirs to get all fasta files.
-        for root, _, files in os.walk(unzipped_path / folder):
-            for file in files:
-                file_ending = file.split(".")[-1]
-                if file_ending in fasta_endings:
-                    species_files.append(Path(root) / file)
-                    all_assemblies.append(".".join(str(file).split(".")[:-1]))
-
-        # Gather all sequences and headers.
-        sequences = []
-        headers = []
-        for file in species_files:
-            records = SeqIO.parse(file, "fasta")
-            for record in records:
-                headers.append(record.id)
-                sequences.append(str(record.seq))
-
-        # Concatenate sequences
-        species_sequence = "".join(sequences)
-        species_header = ">" + " § ".join(headers) + "\n"
-
-        # Save concatenated sequences and headers
-        species_path = concatenate_path / (folder + ".fasta")
-        with open(species_path, "w", encoding="utf-8") as species_file:
-            species_file.write(species_header)
-            species_file.write(species_sequence)
-
-    return all_assemblies
-
-
-def save_all_assemblies(dir_path: Path, all_assemblies: list[str]):
-    """Save all assemblies to a file."""
-    path = dir_path / "all_bf_assemblies.txt"
-    with open(path, "w", encoding="utf-8") as file:
-        for assembly in all_assemblies:
-            file.write(f"{assembly}\n")
-
-
-def bf(dir_name: str, delete: bool):
-    """Extract and concatenate assemblies for Bloom filter training."""
-    dir_path = get_xspect_tmp_path() / dir_name
-    zip_path = dir_path / "zip_files"
-    unzipped_path = dir_path / "zip_files_extracted"
-    concatenate_path = dir_path / "concatenate"
-    file_io.extract_zip(zip_path, unzipped_path)
-    all_assemblies = concatenate_bf(unzipped_path, concatenate_path)
-    save_all_assemblies(dir_path, all_assemblies)
-    if delete:
-        file_io.delete_zip_files(zip_path)
-        shutil.rmtree(zip_path, ignore_errors=False)
-        shutil.rmtree(unzipped_path, ignore_errors=False)
-
-
-def svm(species_accessions: dict, dir_name: str, delete: bool):
-    """Extract and concatenate assemblies for generating SVM training data."""
-    dir_path = get_xspect_tmp_path() / dir_name
-    zip_path = dir_path / "training_data_zipped"
-    unzipped_path = dir_path / "training_data_unzipped"
-    assemblies_path = dir_path / "training_data"
-    file_io.extract_zip(zip_path, unzipped_path)
-    copy_assemblies(unzipped_path, assemblies_path)
-    change_header(assemblies_path, species_accessions)
-    if delete:
-        file_io.delete_zip_files(zip_path)
-        shutil.rmtree(zip_path, ignore_errors=False)
-        shutil.rmtree(unzipped_path, ignore_errors=False)

xspect/train_filter/ncbi_api/download_assemblies.py

@@ -1,31 +0,0 @@
-"""This module contains methods to download assemblies from the NCBI database."""
-
-__author__ = "Berger, Phillip"
-
-# pylint: disable=line-too-long
-
-import os
-import requests
-from xspect.definitions import get_xspect_tmp_path
-
-
-def download_assemblies(accessions, dir_name, target_folder, zip_file_name):
-    """Download and save assemblies from the NCBI database.
-
-    :param accessions: All collected accessions from the NCBI RefSeq-database.
-    :type accessions: list
-    :param dir_name: Name of the directory where the assemblies will be saved.
-    :type dir_name: str
-    :param target_folder: Name for the folder in which the downloaded files will be stored.
-    :type target_folder: str
-    :param zip_file_name: Name of the zip file. E.g. Klebsiella aerogenes.zip.
-    :type zip_file_name: str
-    """
-
-    path = get_xspect_tmp_path() / dir_name / target_folder / zip_file_name
-    api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v2/genome/accession/{','.join(accessions)}/download"
-    parameters = {"include_annotation_type": "GENOME_FASTA", "filename": zip_file_name}
-    os.makedirs(os.path.dirname(path), exist_ok=True)
-    genome_download = requests.get(api_url, params=parameters, timeout=30)
-    with open(path, "wb") as f:
-        f.write(genome_download.content)

xspect/train_filter/ncbi_api/ncbi_assembly_metadata.py

@@ -1,110 +0,0 @@
-"""Collects metadata of assemblies from NCBI API"""
-
-__author__ = "Berger, Phillip"
-
-from time import sleep
-
-import requests
-
-from loguru import logger
-
-
-class NCBIAssemblyMetadata:
-    """Class to collect metadata of assemblies from the NCBI API."""
-
-    _all_metadata: dict
-    _count: int
-    _parameters: dict
-    _accessions: list[str]
-    _contig_n50: int
-    _all_metadata_complete: dict
-
-    def __init__(self, all_metadata: dict, count=8, contig_n50=10000):
-        self._all_metadata = all_metadata
-        self._count = count
-        self._contig_n50 = contig_n50
-
-        self._set_parameters()
-
-        tmp_metadata = {}
-        for tax_id, curr_metadata in self._all_metadata.items():
-            sleep(2)
-            species_name = curr_metadata["sci_name"]
-            logger.info("Collecting metadata of {name}", name=species_name)
-            accessions = self._make_request(taxon=tax_id)
-            if len(accessions) != 0:
-                curr_metadata["accessions"] = accessions
-                tmp_metadata[tax_id] = curr_metadata
-
-        self._all_metadata_complete = tmp_metadata
-
-    def _set_parameters(self):
-        params = {
-            "filters.reference_only": "false",
-            "filters.assembly_source": "refseq",
-            "filters.exclude_atypical": "true",
-            "page_size": self._count,
-            "page_token": "",
-        }
-        params_ref = params.copy()
-        params_ref["filters.reference_only"] = "true"
-
-        params_comp_genome = params.copy()
-        params_comp_genome["filters.assembly_level"] = "complete_genome"
-
-        params_chrom = params.copy()
-        params_chrom["filters.assembly_level"] = "chromosome"
-
-        params_scaffold = params.copy()
-        params_scaffold["filters.assembly_level"] = "scaffold"
-
-        params_contig = params.copy()
-        params_contig["filters.assembly_level"] = "contig"
-
-        self._parameters = {
-            "params_ref": params_ref,
-            "params_comp_genome": params_comp_genome,
-            "params_chrom": params_chrom,
-            "params_scaffold": params_scaffold,
-            "params_contig": params_contig,
-        }
-
-    def _make_request(self, taxon: str):
-        api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v2/genome/taxon/{taxon}/dataset_report"
-        accessions = []
-        count = 0
-        for request_type, parameters in self._parameters.items():
-            raw_response = requests.get(api_url, params=parameters, timeout=5)
-            response = raw_response.json()
-            if response:
-                try:
-                    reports = response["reports"]
-                    for report in reports:
-                        accession = report["accession"]
-                        contig_n50 = report["assembly_stats"]["contig_n50"]
-                        taxonomy_check_status = report["average_nucleotide_identity"][
-                            "taxonomy_check_status"
-                        ]
-                        if count < self._count:
-                            if (
-                                taxonomy_check_status == "OK"
-                                and contig_n50 > self._contig_n50
-                            ):
-                                accessions.append(accession)
-                                count += 1
-                        else:
-                            break
-                except KeyError:
-                    logger.debug(
-                        "While requesting: {type} an error response was given",
-                        type=request_type,
-                    )
-                    logger.debug(str(response))
-
-            if count >= self._count:
-                break
-        return accessions
-
-    def get_all_metadata(self):
-        """Returns all metadata of the assemblies."""
-        return self._all_metadata_complete

xspect/train_filter/ncbi_api/ncbi_children_tree.py

@@ -1,53 +0,0 @@
-"""This class uses the NCBI Datasets API to get the taxonomy tree of a given Taxon.
-
-The taxonomy tree consists of only the next children to the parent taxon.
-The children are only of the next lower rank of the parent taxon.
-"""
-
-__author__ = "Berger, Phillip"
-
-import sys
-import requests
-
-from loguru import logger
-
-
-class NCBIChildrenTree:
-    _taxon: str
-    _response: dict
-    _parent_taxon_id: str
-    _children_taxon_ids = list()
-
-    def __init__(self, taxon: str):
-        self._taxon = taxon
-        self._request_tree()
-
-    def _request_tree(self):
-        """Make the request for the children tree at the NCBI Datasets API."""
-        api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v2/taxonomy/taxon/{self._taxon}/filtered_subtree"
-        raw_response = requests.get(api_url, timeout=5)
-        self._response = raw_response.json()["edges"]
-        self._parent_taxon_id = str(self._response["1"]["visible_children"][0])
-        try:
-            tmp_children_ids = self._response[self._parent_taxon_id]["visible_children"]
-        except KeyError:
-            logger.error("KeyError for key: {key}", key=self._parent_taxon_id)
-            logger.error("Response: {response}", response=str(self._response))
-            logger.error("Aborting")
-            sys.exit()
-        for child_id in tmp_children_ids:
-            self._children_taxon_ids.append(str(child_id))
-
-    def parent_id(self):
-        """The NCBI taxon ID of the given Taxon at the initialisation.
-
-        :return: The taxon ID.
-        """
-        return self._parent_taxon_id
-
-    def children_ids(self) -> list[str]:
-        """The NCBI taxon IDs of all children of the given Taxon. The children are all of a lower rank than the parent.
-
-        :return: The taxon IDs as a list.
-        """
-        return self._children_taxon_ids

xspect/train_filter/ncbi_api/ncbi_taxon_metadata.py

@@ -1,55 +0,0 @@
-"""This module is used to retrieve metadata from the NCBI taxonomy database."""
-
-__author__ = "Berger, Phillip"
-
-import requests
-
-from loguru import logger
-
-
-class NCBITaxonMetadata:
-    """Class to retrieve metadata from the NCBI taxonomy database."""
-
-    _taxon: str
-    _response: dict
-    _all_metadata: dict
-
-    def __init__(self, taxon: list[str]):
-        self._taxon = ",".join(taxon)
-        self._all_metadata = {}
-        self._request_metadata()
-        self._collect_all_metadata()
-
-    def _request_metadata(self):
-        api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v2/taxonomy/taxon/{str(self._taxon)}"
-        raw_response = requests.get(api_url, timeout=5)
-        self._response = raw_response.json()["taxonomy_nodes"]
-
-    def _collect_all_metadata(self):
-        for child_metadata in self._response:
-            taxonomy = child_metadata["taxonomy"]
-            rank = taxonomy["rank"]
-            name = taxonomy["organism_name"]
-            tax_id = str(taxonomy["tax_id"])
-            lineage = taxonomy["lineage"]
-            if "Candidatus" not in name:
-                if " sp. " not in name:
-                    metadata = {
-                        "sci_name": name,
-                        "tax_id": tax_id,
-                        "rank": rank,
-                        "lineage": lineage,
-                    }
-                    self._all_metadata[tax_id] = metadata
-                else:
-                    logger.debug("{name} was not used for training", name=name)
-            else:
-                logger.debug("{name} was not used for training", name=name)
-
-    def get_response(self):
-        """Returns the raw response from the NCBI taxonomy database."""
-        return self._response
-
-    def get_metadata(self):
-        """Returns the metadata from the NCBI taxonomy database."""
-        return self._all_metadata