XspecT 0.2.6__py3-none-any.whl → 0.4.0__py3-none-any.whl

XspecT-0.2.6.dist-info/RECORD

@@ -1,34 +0,0 @@
-xspect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/definitions.py,sha256=Z6RiCOQnsA_i8DPIq-7SUhrKo7KYf1Azp36UZZpcOX4,1419
-xspect/download_models.py,sha256=lml8pSyM0pF-MxghgSRC9noDT4pkUcBZraaTTImVfbA,739
-xspect/fastapi.py,sha256=FDiGXJmLEeTLD83Hem8yV5aoPJ-GhSG5WmDIQRAA_w4,3257
-xspect/file_io.py,sha256=zKhl6Fd9KZAYiD8YgIyje5TbDYk5lxMp1WUrNkGSBo8,2779
-xspect/main.py,sha256=3HqmnMowjkLNwhaZWtY4aeJCyCyT6h_nZWzYIunHfKg,5325
-xspect/model_management.py,sha256=xF-wjVNJbXYv64RajsIcpLfZUvicDyalJEdSeCx3nQI,3542
-xspect/pipeline.py,sha256=h7duhVZ-hupwO_KQPstzFo8KMfMI2yleb9HmtTiMjic,7219
-xspect/run.py,sha256=OJ7pCFqva3AhIYklKjVnqWGooVRO7S3b56kIAy-xabY,1189
-xspect/train.py,sha256=p_5BPh7XNA7R2h8MwpN0-AwzjbNIxdmeMKztP7RU4g8,9499
-xspect/mlst_feature/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/mlst_feature/mlst_helper.py,sha256=omqKmreah5qaspnJ5qKp_9oZsdHxi0tUJnEzZbpWPEw,5916
-xspect/mlst_feature/pub_mlst_handler.py,sha256=oss3CkJNt6041p3qnMdOfoX8ZgUfpB93CUim-Yakc9A,5031
-xspect/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/models/probabilistic_filter_mlst_model.py,sha256=JMc0yBJPo7J9b-GpvhDmzhwWPIKOwatAq0edDgM72PE,11735
-xspect/models/probabilistic_filter_model.py,sha256=zCn5dcuq5Z4pvmsV9igS0lQ1plUi9-Kky_zRflfrIkI,9659
-xspect/models/probabilistic_filter_svm_model.py,sha256=uabDrF1_CSuIWf9wWyQAkqjAuRUBzEZLkv3J6YHfJsM,5641
-xspect/models/probabilistic_single_filter_model.py,sha256=TdGbQp8ylOif7dD13OSWaS-zFNJo8mXOb6BaQ0mcPdo,3810
-xspect/models/result.py,sha256=RuYqagyG5QbFlW408haXzDbPYJ6yS2flcWcJ8Dy6UjM,4834
-xspect/train_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/train_filter/create_svm.py,sha256=w6gq40yHINVfNzLhJfYFykUaNCwpU9AEDcbkUfis3DY,1504
-xspect/train_filter/extract_and_concatenate.py,sha256=lLrczGgfZi2vAGqxq8fcEmJi5pvqyK33JkB_ZoCNYG8,4840
-xspect/train_filter/html_scrap.py,sha256=76VV_ZbvD2I3IxRb62SiQwRPu2tr4fwn1HkfJQYaosM,3809
-xspect/train_filter/ncbi_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/train_filter/ncbi_api/download_assemblies.py,sha256=MB_mxSjCTL05DqIt1WQem8AGU3PjtJnzPndeI9J-AOI,1285
-xspect/train_filter/ncbi_api/ncbi_assembly_metadata.py,sha256=puzDIws-yyBAEHwSAIYUM7g8FpLFmvOKh5xH1EsY8ZE,3830
-xspect/train_filter/ncbi_api/ncbi_children_tree.py,sha256=_8puOsnsKp5lsMV2gZY1ijkfD_BZKG9eXZCX09qph5E,1819
-xspect/train_filter/ncbi_api/ncbi_taxon_metadata.py,sha256=O6JDXC4E6AYaf7NPnb34eSJyZhMB8r--bjoVF_ZsEdA,1868
-XspecT-0.2.6.dist-info/LICENSE,sha256=bhBGDKIRUVwYIHGOGO5hshzuVHyqFJajvSOA3XXOLKI,1094
-XspecT-0.2.6.dist-info/METADATA,sha256=cO5cpS3zD45dpARRO92XG9bCfCbJ1LE-bAaIxkb0bwQ,4714
-XspecT-0.2.6.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-XspecT-0.2.6.dist-info/entry_points.txt,sha256=L7qliX3pIuwupQxpuOSsrBJCSHYPOPNEzH8KZKQGGUw,43
-XspecT-0.2.6.dist-info/top_level.txt,sha256=hdoa4cnBv6OVzpyhMmyxpJxEydH5n2lDciy8urc1paE,7
-XspecT-0.2.6.dist-info/RECORD,,

xspect/pipeline.py

@@ -1,201 +0,0 @@
-""" Module for defining the Pipeline class. """
-
-import json
-from pathlib import Path
-from Bio.SeqRecord import SeqRecord
-from Bio import SeqIO
-from xspect.file_io import get_records_by_id
-from xspect.models.result import StepType, SubprocessingStep
-from xspect.run import Run
-from xspect.models.result import ModelResult
-from xspect.model_management import get_model_by_slug
-
-
-class ModelExecution:
-    """Class for storing a processing step of an XspecT pipeline."""
-
-    def __init__(
-        self,
-        model_slug: str,
-        sparse_sampling_step: int = 1,
-    ):
-        self.model_slug = model_slug
-        self.sparse_sampling_step = sparse_sampling_step
-        self.pipeline_steps = []
-
-    def add_pipeline_step(
-        self,
-        pipeline_step: "PipelineStep",
-    ):
-        """Add a subprocessing step to the pipeline step."""
-        self.pipeline_steps.append(pipeline_step)
-
-    def to_dict(self) -> dict:
-        """Return the processing step as a dictionary."""
-        return {
-            "model_slug": self.model_slug,
-            "sparse_sampling_step": self.sparse_sampling_step,
-            "pipeline_steps": [
-                pipeline_step.to_dict() for pipeline_step in self.pipeline_steps
-            ],
-        }
-
-    def run(
-        self,
-        sequence_input: (
-            SeqRecord
-            | list[SeqRecord]
-            | SeqIO.FastaIO.FastaIterator
-            | SeqIO.QualityIO.FastqPhredIterator
-            | Path
-        ),
-    ) -> ModelResult:
-        """Run the model on a given input."""
-        model = get_model_by_slug(self.model_slug)
-        model_result = model.predict(sequence_input, step=self.sparse_sampling_step)
-
-        for pipeline_step in self.pipeline_steps:
-            if pipeline_step.subprocessing_type == StepType.PREDICTION:
-                score = model_result.get_scores()["total"][pipeline_step.label]
-                if score >= pipeline_step.treshold:
-                    prediction_model_result = pipeline_step.model_execution.run(
-                        sequence_input
-                    )
-                    subprocessing_step = SubprocessingStep(
-                        pipeline_step.subprocessing_type,
-                        pipeline_step.label,
-                        pipeline_step.treshold,
-                        prediction_model_result,
-                    )
-                    model_result.add_subprocessing_step(subprocessing_step)
-            elif pipeline_step.subprocessing_type == StepType.FILTERING:
-                filtered_sequence_ids = model_result.get_filtered_subsequences(
-                    pipeline_step.label, pipeline_step.treshold
-                )
-                sequence_input = get_records_by_id(
-                    sequence_input, filtered_sequence_ids
-                )
-
-                filtering_model_result = None
-                if sequence_input:
-                    filtering_model_result = pipeline_step.model_execution.run(
-                        sequence_input
-                    )
-
-                subprocessing_step = SubprocessingStep(
-                    pipeline_step.subprocessing_type,
-                    pipeline_step.label,
-                    pipeline_step.treshold,
-                    filtering_model_result,
-                )
-                model_result.add_subprocessing_step(subprocessing_step)
-            else:
-                raise ValueError(
-                    f"Invalid subprocessing type {pipeline_step.subprocessing_type}"
-                )
-
-        return model_result
-
-
-class PipelineStep:
-    """Class for storing a subprocessing step of an XspecT model."""
-
-    def __init__(
-        self,
-        subprocessing_type: StepType,
-        label: str,
-        treshold: float,
-        model_execution: ModelExecution,
-    ):
-        self.subprocessing_type = subprocessing_type
-        self.label = label
-        self.treshold = treshold
-        self.model_execution = model_execution
-
-    def to_dict(self) -> dict:
-        """Return the subprocessing step as a dictionary."""
-        return {
-            "subprocessing_type": str(self.subprocessing_type),
-            "label": self.label,
-            "treshold": self.treshold,
-            "model_execution": self.model_execution.to_dict(),
-        }
-
-
-class Pipeline:
-    """Class for storing an XspecT pipeline consisting of multiple model processing steps."""
-
-    def __init__(self, display_name: str, author: str, author_email: str):
-        self.display_name = display_name
-        self.author = author
-        self.author_email = author_email
-        self.model_executions = []
-
-    def add_pipeline_step(
-        self,
-        pipeline_step: ModelExecution,
-    ):
-        """Add a processing step to the pipeline."""
-        self.model_executions.append(pipeline_step)
-
-    def to_dict(self) -> dict:
-        """Return the pipeline as a dictionary."""
-        return {
-            "display_name": self.display_name,
-            "author": self.author,
-            "author_email": self.author_email,
-            "model_executions": [
-                model_execution.to_dict() for model_execution in self.model_executions
-            ],
-        }
-
-    def to_json(self) -> str:
-        """Return the pipeline as a JSON string."""
-        return json.dumps(self.to_dict())
-
-    def save(self, path: Path) -> None:
-        """Save the pipeline as a JSON file."""
-        with open(path, "w", encoding="utf-8") as f:
-            f.write(self.to_json())
-
-    @staticmethod
-    def from_file(path: Path) -> "Pipeline":
-        """Load the pipeline from a JSON file."""
-        with open(path, "r", encoding="utf-8") as f:
-            pipeline_json = json.load(f)
-            pipeline = Pipeline(
-                pipeline_json["display_name"],
-                pipeline_json["author"],
-                pipeline_json["author_email"],
-            )
-            for model_execution in pipeline_json["model_executions"]:
-                model_execution = ModelExecution(
-                    model_execution["model_slug"],
-                    model_execution["sparse_sampling_step"],
-                )
-                for pipeline_step in model_execution["pipeline_steps"]:
-                    model_execution.add_pipeline_step(
-                        PipelineStep(
-                            StepType(pipeline_step["subprocessing_type"]),
-                            pipeline_step["label"],
-                            pipeline_step["treshold"],
-                            ModelExecution(
-                                pipeline_step["model_execution"]["model_slug"],
-                                pipeline_step["model_execution"][
-                                    "sparse_sampling_step"
-                                ],
-                            ),
-                        )
-                    )
-                pipeline.add_pipeline_step(model_execution)
-            return pipeline
-
-    def run(self, input_file: Path) -> Run:
-        """Run the pipeline on a given input."""
-        run = Run(self.display_name, input_file)
-
-        for model_execution in self.model_executions:
-            result = model_execution.run(input_file)
-            run.add_result(result)
-
-        return run

xspect/run.py

@@ -1,38 +0,0 @@
-""" Module with XspecT global run class, which summarizes individual model results. """
-
-import json
-from pathlib import Path
-from xspect.models.result import ModelResult
-
-
-class Run:
-    """Class for storing the results of an XspecT run."""
-
-    def __init__(self, display_name: str, input_file: str):
-        self.display_name = display_name
-        self.input_file = input_file
-        self.results = []
-
-    def add_result(self, result: ModelResult):
-        """Add a result to the run."""
-        self.results.append(result)
-
-    def to_dict(self) -> dict:
-        """Return the run as a dictionary."""
-        return {
-            "display_name": self.display_name,
-            "input_file": str(self.input_file),
-            "results": (
-                [result.to_dict() for result in self.results] if self.results else []
-            ),
-        }
-
-    def to_json(self) -> str:
-        """Return the run as a JSON string."""
-        json_dict = self.to_dict()
-        return json.dumps(json_dict, indent=4)
-
-    def save(self, path: Path) -> None:
-        """Save the run as a JSON file."""
-        with open(path, "w", encoding="utf-8") as f:
-            f.write(self.to_json())

xspect/train_filter/create_svm.py

@@ -1,45 +0,0 @@
-"""This module contains functions to select and download assemblies for SVM creation."""
-
-from time import sleep
-from loguru import logger
-from xspect.train_filter.ncbi_api import download_assemblies
-
-
-def select_assemblies(accessions):
-    """Selects up to 4 assemblies, ideally assemblies that were not used for training the filters.
-
-    :param accessions: All selected assembly accessions for every species.
-    :type accessions: dict
-    :return: Dict with species name as key and selected accessions as value.
-    """
-
-    all_accessions = {
-        sci_name: curr_accessions[-4:]
-        for sci_name, curr_accessions in accessions.items()
-    }
-
-    return all_accessions
-
-
-def get_svm_assemblies(all_accessions, dir_name):
-    """Download assemblies for svm creation.
-
-    :param all_accessions: Contains lists with all previously selected assemblies for every species.
-    :type all_accessions: dict
-    :param dir_name: Name of the parent directory.
-    :type dir_name: str
-    """
-    # Select accessions for download.
-    selected_accessions = select_assemblies(all_accessions)
-
-    # Download assemblies.
-    for sci_name, accessions in selected_accessions.items():
-        sleep(5)
-        logger.info("Downloading {name}", name=sci_name)
-        file_name = sci_name + ".zip"
-        download_assemblies.download_assemblies(
-            accessions=accessions,
-            dir_name=dir_name,
-            target_folder="training_data_zipped",
-            zip_file_name=file_name,
-        )

xspect/train_filter/extract_and_concatenate.py

@@ -1,124 +0,0 @@
-""" Module for extracting and concatenating assemblies. """
-
-__author__ = "Berger, Phillip"
-
-import os
-import shutil
-from pathlib import Path
-from Bio import SeqIO
-from xspect import file_io
-from xspect.definitions import get_xspect_tmp_path, fasta_endings
-
-
-def change_header(assemblies_path, species_accessions: dict):
-    """Change the header of the assemblies to the species name."""
-    files = os.listdir(assemblies_path)
-    # Iterate through all species.
-    for name, accessions in species_accessions.items():
-        # Iterate through all accessions of the current species.
-        for accession in accessions:
-            # Iterate through all file names.
-            for file in files:
-                if accession in file:
-                    file_path = assemblies_path / str(file)
-                    # Change the header.
-                    with open(file_path, "r", encoding="utf-8") as f:
-                        sequence = ""
-                        for line in f.readlines():
-                            if line[0] != ">":
-                                sequence += line
-                        new_header = f">{name}\n"
-                    with open(file_path, "w", encoding="utf-8") as f:
-                        f.write(new_header)
-                        f.write(sequence)
-
-
-def copy_assemblies(unzipped_path, assemblies_path):
-    """Copy all assemblies to a new directory."""
-    os.mkdir(assemblies_path)
-    for folder in os.listdir(unzipped_path):
-        for root, _, files in os.walk(unzipped_path / str(folder)):
-            for file in files:
-                file_ending = file.split(".")[-1]
-                if file_ending in fasta_endings:
-                    file_path = Path(root) / file
-                    shutil.copy(file_path, (assemblies_path / file))
-
-
-def concatenate_bf(unzipped_path, concatenate_path):
-    """Concatenate all assemblies for Bloom filter training."""
-
-    all_assemblies = []
-
-    # Make new directory
-    os.mkdir(concatenate_path)
-
-    # Open the fasta files for each species.
-    for folder in os.listdir(unzipped_path):
-        species_files = []
-        # Walk through dirs to get all fasta files.
-        for root, _, files in os.walk(unzipped_path / folder):
-            for file in files:
-                file_ending = file.split(".")[-1]
-                if file_ending in fasta_endings:
-                    species_files.append(Path(root) / file)
-                    all_assemblies.append(".".join(str(file).split(".")[:-1]))
-
-        # Gather all sequences and headers.
-        sequences = []
-        headers = []
-        for file in species_files:
-            records = SeqIO.parse(file, "fasta")
-            for record in records:
-                headers.append(record.id)
-                sequences.append(str(record.seq))
-
-        # Concatenate sequences
-        species_sequence = "".join(sequences)
-        species_header = ">" + " § ".join(headers) + "\n"
-
-        # Save concatenated sequences and headers
-        species_path = concatenate_path / (folder + ".fasta")
-        with open(species_path, "w", encoding="utf-8") as species_file:
-            species_file.write(species_header)
-            species_file.write(species_sequence)
-
-    return all_assemblies
-
-
-def save_all_assemblies(dir_path: Path, all_assemblies: list[str]):
-    """Save all assemblies to a file."""
-    path = dir_path / "all_bf_assemblies.txt"
-    with open(path, "w", encoding="utf-8") as file:
-        for assembly in all_assemblies:
-            file.write(f"{assembly}\n")
-
-
-def bf(dir_name: str, delete: bool):
-    """Extract and concatenate assemblies for Bloom filter training."""
-    dir_path = get_xspect_tmp_path() / dir_name
-    zip_path = dir_path / "zip_files"
-    unzipped_path = dir_path / "zip_files_extracted"
-    concatenate_path = dir_path / "concatenate"
-    file_io.extract_zip(zip_path, unzipped_path)
-    all_assemblies = concatenate_bf(unzipped_path, concatenate_path)
-    save_all_assemblies(dir_path, all_assemblies)
-    if delete:
-        file_io.delete_zip_files(zip_path)
-        shutil.rmtree(zip_path, ignore_errors=False)
-        shutil.rmtree(unzipped_path, ignore_errors=False)
-
-
-def svm(species_accessions: dict, dir_name: str, delete: bool):
-    """Extract and concatenate assemblies for generating SVM training data."""
-    dir_path = get_xspect_tmp_path() / dir_name
-    zip_path = dir_path / "training_data_zipped"
-    unzipped_path = dir_path / "training_data_unzipped"
-    assemblies_path = dir_path / "training_data"
-    file_io.extract_zip(zip_path, unzipped_path)
-    copy_assemblies(unzipped_path, assemblies_path)
-    change_header(assemblies_path, species_accessions)
-    if delete:
-        file_io.delete_zip_files(zip_path)
-        shutil.rmtree(zip_path, ignore_errors=False)
-        shutil.rmtree(unzipped_path, ignore_errors=False)

xspect/train_filter/html_scrap.py

@@ -1,114 +0,0 @@
-""" HTML Scraping for the taxonomy check results from NCBI."""
-
-__author__ = "Berger, Phillip"
-
-import datetime
-import pickle
-import sys
-import time
-import requests
-from loguru import logger
-from xspect.definitions import get_xspect_root_path
-
-
-class TaxonomyCheck:
-    """Class to get the GCFs that passed the taxonomy check from NCBI."""
-
-    _main_url = "https://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/ANI_report_prokaryotes.txt"
-    _test_url = "https://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/"
-    _main_path = get_xspect_root_path() / "taxonomy_check.txt"
-    _test_path = get_xspect_root_path() / "tax_check_date.txt"
-    _new_time: list
-    _tax_check_ok = []
-
-    def __init__(self):
-        old_time = self._get_old_tax_date()
-        self._new_time = self._get_new_tax_date()
-        # Both Dates could be found.
-        # Check if the html file was updated since the last time it was downloaded.
-        # If yes than update the file.
-        if self._new_time and old_time:
-            if self._new_time == old_time:
-                logger.info("File was not updated")
-                self._get_old_file()
-            else:
-                logger.info("Updating file")
-                self._update_tax_check()
-
-        # The old date does not exist.
-        # Get the html file for the taxonomy check results.
-        elif self._new_time and not old_time:
-            logger.info("No file was found. Creating new file")
-            self._update_tax_check()
-
-        elif not self._new_time and old_time:
-            self._get_old_file()
-
-        else:
-            logger.error("Nothing was found")
-            logger.error("Aborting")
-            sys.exit()
-
-    def _get_old_tax_date(self):
-        try:
-            with open(self._test_path, "rb") as f:
-                old_time = pickle.load(f)
-                return old_time
-        except FileNotFoundError:
-            return None
-
-    def _get_new_tax_date(self):
-        raw_response = requests.get(self._test_url, timeout=5)
-        data = raw_response.text.split("\n")
-        for line in data:
-            if "ANI_report_prokaryotes.txt" in line:
-                line_parts = line.split()
-                date_parts = line_parts[-3].split("-")
-                date = datetime.date(
-                    int(date_parts[0]), int(date_parts[1]), int(date_parts[2])
-                )
-                time_parts = line_parts[-2].split(":")
-                combined_time = datetime.time(int(time_parts[0]), int(time_parts[1]))
-                new_time = [date, combined_time]
-
-                return new_time
-
-        return None
-
-    def _update_tax_check(self):
-        raw_response = requests.get(self._main_url, timeout=5)
-        all_tax_checks = raw_response.text.split("\n")[1:-1]
-        self._get_gcf_ok(all_tax_checks)
-        self._save_time()
-        self._save_file()
-
-    def _get_gcf_ok(self, all_tax_checks: list):
-        tax_check_ok = []
-        for line in all_tax_checks:
-            line_parts = line.split("\t")
-            gcf = line_parts[1]
-            tax_check_status = line_parts[-1]
-            if tax_check_status == "OK":
-                tax_check_ok.append(gcf)
-
-        self._tax_check_ok = tax_check_ok
-
-    def _save_time(self):
-        with open(self._test_path, "wb") as f:
-            pickle.dump(self._new_time, f)
-
-    def _save_file(self):
-        with open(self._main_path, "wb") as f:
-            pickle.dump(self._tax_check_ok, f)
-
-    def _get_old_file(self):
-        with open(self._main_path, "rb") as f:
-            self._tax_check_ok = pickle.load(f)
-
-    @staticmethod
-    def _get_current_time():
-        return time.asctime(time.localtime()).split()[3]
-
-    def ani_gcf(self):
-        """Returns ANI GCFs that passed the taxonomy check."""
-        return self._tax_check_ok

xspect/train_filter/ncbi_api/download_assemblies.py

@@ -1,31 +0,0 @@
-"""This module contains methods to download assemblies from the NCBI database."""
-
-__author__ = "Berger, Phillip"
-
-# pylint: disable=line-too-long
-
-import os
-import requests
-from xspect.definitions import get_xspect_tmp_path
-
-
-def download_assemblies(accessions, dir_name, target_folder, zip_file_name):
-    """Download and save assemblies from the NCBI database.
-
-    :param accessions: All collected accessions from the NCBI RefSeq-database.
-    :type accessions: list
-    :param dir_name: Name of the directory where the assemblies will be saved.
-    :type dir_name: str
-    :param target_folder: Name for the folder in which the downloaded files will be stored.
-    :type target_folder: str
-    :param zip_file_name: Name of the zip file. E.g. Klebsiella aerogenes.zip.
-    :type zip_file_name: str
-    """
-
-    path = get_xspect_tmp_path() / dir_name / target_folder / zip_file_name
-    api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v1/genome/accession/{','.join(accessions)}/download"
-    parameters = {"include_annotation_type": "GENOME_FASTA", "filename": zip_file_name}
-    os.makedirs(os.path.dirname(path), exist_ok=True)
-    genome_download = requests.get(api_url, params=parameters, timeout=20)
-    with open(path, "wb") as f:
-        f.write(genome_download.content)