XspecT 0.5.4__py3-none-any.whl → 0.6.0__py3-none-any.whl

xspect/classify.py

@@ -46,6 +46,7 @@ def classify_species(
     output_path: Path,
     step: int = 1,
     display_name: bool = False,
+    validation: bool = False,
 ):
     """
     Classify the species of sequences.
@@ -59,6 +60,7 @@ def classify_species(
         output_path (Path): The path to the output file where results will be saved.
         step (int): The amount of kmers to be skipped.
         display_name (bool): Includes a display name for each tax_ID.
+        validation (bool): Sorts out misclassified reads.
     """
     ProbabilisticFilterSVMModel = import_module(
         "xspect.models.probabilistic_filter_svm_model"
@@ -69,7 +71,12 @@ def classify_species(
     input_paths, get_output_path = prepare_input_output_paths(input_path)
 
     for idx, current_path in enumerate(input_paths):
-        result = model.predict(current_path, step=step, display_name=display_name)
+        result = model.predict(
+            current_path,
+            step=step,
+            display_name=display_name,
+            validation=validation,
+        )
         result.input_source = current_path.name
         cls_path = get_output_path(idx, output_path)
         result.save(cls_path)

xspect/definitions.py

@@ -89,3 +89,22 @@ def get_xspect_mlst_path() -> Path:
     mlst_path = get_xspect_root_path() / "mlst"
     mlst_path.mkdir(exist_ok=True, parents=True)
     return mlst_path
+
+
+def get_xspect_misclassification_path() -> Path:
+    """
+    Notes:
+    Developed by Oemer Cetin as part of a Bsc thesis at Goethe University Frankfurt am Main (2025).
+    (An Integration of Alignment-Free and Alignment-Based Approaches for Bacterial Taxon Assignment)
+
+    Return the path to the XspecT Misclassification directory.
+
+    Returns the path to the XspecT Misclassification directory, which is located within the XspecT data
+    directory. If the directory does not exist, it creates the directory.
+
+    Returns:
+        Path: The path to the XspecT Misclassification directory.
+    """
+    misclassification_path = get_xspect_root_path() / "misclassification"
+    misclassification_path.mkdir(exist_ok=True, parents=True)
+    return misclassification_path

xspect/main.py

@@ -87,13 +87,62 @@ def train():
     help="Email of the author.",
     default=None,
 )
-def train_ncbi(model_genus, svm_steps, author, author_email):
+@click.option(
+    "--min-n50",
+    type=int,
+    help="Minimum contig N50 to filter the accessions (default: 10000).",
+    default=10000,
+)
+@click.option(
+    "--include-atypical/--exclude-atypical",
+    help="Include or exclude atypical accessions (default: exclude).",
+    default=False,
+)
+@click.option(
+    "--allow-inconclusive",
+    is_flag=True,
+    help="Allow the use of accessions with inconclusive taxonomy check status for training.",
+    default=False,
+)
+@click.option(
+    "--allow-candidatus",
+    is_flag=True,
+    help="Allow the use of Candidatus species for training.",
+    default=False,
+)
+@click.option(
+    "--allow-sp",
+    is_flag=True,
+    help="Allow the use of species with 'sp.' in their names for training.",
+    default=False,
+)
+def train_ncbi(
+    model_genus,
+    svm_steps,
+    author,
+    author_email,
+    min_n50,
+    include_atypical,
+    allow_inconclusive,
+    allow_candidatus,
+    allow_sp,
+):
     """Train a species and a genus model based on NCBI data."""
     click.echo(f"Training {model_genus} species and genus metagenome model.")
     try:
         train_from_ncbi = import_module("xspect.train").train_from_ncbi
 
-        train_from_ncbi(model_genus, svm_steps, author, author_email)
+        train_from_ncbi(
+            model_genus,
+            svm_steps,
+            author,
+            author_email,
+            min_n50=min_n50,
+            exclude_atypical=not include_atypical,
+            allow_inconclusive=allow_inconclusive,
+            allow_candidatus=allow_candidatus,
+            allow_sp=allow_sp,
+        )
     except ValueError as e:
         click.echo(f"Error: {e}")
         return
@@ -287,8 +336,19 @@ def classify_genus(model_genus, input_path, output_path, sparse_sampling_step):
     help="Includes the display names next to taxonomy-IDs.",
     is_flag=True,
 )
+@click.option(
+    "-v",
+    "--validation",
+    help="Detects misclassification for small reads or contigs.",
+    is_flag=True,
+)
 def classify_species(
-    model_genus, input_path, output_path, sparse_sampling_step, display_names
+    model_genus,
+    input_path,
+    output_path,
+    sparse_sampling_step,
+    display_names,
+    validation,
 ):
     """Classify samples using a species model."""
     click.echo("Classifying...")
@@ -300,6 +360,7 @@ def classify_species(
         Path(output_path),
         sparse_sampling_step,
         display_names,
+        validation,
     )
 
 

xspect/misclassification_detection/mapping.py

@@ -0,0 +1,168 @@
+"""
+Mapping handler for the alignment-based misclassification detection.
+
+Notes:
+Developed by Oemer Cetin as part of a Bsc thesis at Goethe University Frankfurt am Main (2025).
+(An Integration of Alignment-Free and Alignment-Based Approaches for Bacterial Taxon Assignment)
+"""
+
+import mappy, pysam, os, csv
+from Bio import SeqIO
+from xspect.definitions import fasta_endings
+
+__author__ = "Cetin, Oemer"
+
+
+class MappingHandler:
+    """Handler class for all mapping related procedures."""
+
+    def __init__(self, ref_genome_path: str, reads_path: str) -> None:
+        """
+        Initialise the mapping handler.
+
+        This method sets up the paths to the reference genome and query sequences.
+        Additionally, the paths to the output formats (SAM, BAM and TSV) are generated.
+
+        Args:
+            ref_genome_path (str): The path to the reference genome.
+            reads_path (str): The path to the query sequences.
+        """
+        if not os.path.isfile(ref_genome_path):
+            raise ValueError("The path to the reference genome does not exist.")
+
+        if not os.path.isfile(reads_path):
+            raise ValueError("The path to the reads does not exist.")
+
+        if not ref_genome_path.endswith(tuple(fasta_endings)) and reads_path.endswith(
+            tuple(fasta_endings)
+        ):
+            raise ValueError("The files must be FASTA-files!")
+
+        stem = reads_path.rsplit(".", 1)[0] + "_mapped"
+        self.ref_genome_path = ref_genome_path
+        self.reads_path = reads_path
+        self.sam = stem + ".sam"
+        self.bam = stem + ".sorted.bam"
+        self.tsv = stem + ".start_coordinates.tsv"
+
+    def map_reads_onto_reference(self) -> None:
+        """
+        A Method that maps reads against the respective reference genome.
+
+        This function creates a SAM file via Mappy and converts it into a BAM file.
+        """
+        # create header (entry = sequences of the reference genome)
+        ref_seq = [
+            {"SN": rec.id, "LN": len(rec.seq)}
+            for rec in SeqIO.parse(self.ref_genome_path, "fasta")
+        ]
+        header = {"HD": {"VN": "1.0"}, "SQ": ref_seq}
+        target_id = {sequence["SN"]: number for number, sequence in enumerate(ref_seq)}
+
+        reads = list(SeqIO.parse(self.reads_path, "fasta"))
+        if not reads:
+            raise ValueError("Reads file is empty.")
+
+        read_length = len(reads[0].seq)
+        preset = "map-ont" if read_length > 150 else "sr"
+        # create SAM-file
+        aln = mappy.Aligner(self.ref_genome_path, preset=preset)
+        with pysam.AlignmentFile(self.sam, "w", header=header) as out:
+            for read in reads:
+                read_seq = str(read.seq)
+                for hit in aln.map(read_seq):
+                    if hit.cigar_str is None:
+                        continue
+                    # add soft-clips so CIGAR length == len(read_seq) IMPORTANT!!
+                    leftS = hit.q_st
+                    rightS = len(read_seq) - hit.q_en
+                    cigar = (
+                        (f"{leftS}S" if leftS > 0 else "")
+                        + hit.cigar_str
+                        + (f"{rightS}S" if rightS > 0 else "")
+                    )
+
+                    mapped_region = pysam.AlignedSegment()
+                    mapped_region.query_name = read.id
+                    mapped_region.query_sequence = read_seq
+                    mapped_region.flag = 16 if hit.strand == -1 else 0
+                    mapped_region.reference_id = target_id[hit.ctg]
+                    mapped_region.reference_start = hit.r_st
+                    mapped_region.mapping_quality = (
+                        hit.mapq or 255
+                    )  # 0-60 (255 means unavailable)
+                    mapped_region.cigarstring = cigar
+                    out.write(mapped_region)
+                    break  # keep only primary
+
+        # create BAM-file
+        pysam.sort("-o", self.bam, self.sam)
+        pysam.index(self.bam)
+
+    def get_total_genome_length(self) -> int:
+        """
+        Get the genome length from a BAM-file.
+
+        This function opens a BAM-file and extracts the genome length information.
+
+        Returns:
+            int: The genome length.
+        """
+        with pysam.AlignmentFile(self.bam, "rb") as bam:
+            return sum(bam.lengths)
+
+    def extract_starting_coordinates(self) -> None:
+        """
+        Extract starting coordinates of mapped regions from a BAM-file.
+
+        This function scans through a BAM-file and creates a TSV-file.
+        The information that is extracted is the starting coordinate for each mapped read.
+        """
+        # create tsv-file with all start positions
+        with open(self.tsv, "w") as tsv:
+            tsv.write("reference_genome\tread\tmapped_starting_coordinate\n")
+            try:
+                with pysam.AlignmentFile(self.bam, "rb") as bam:
+                    entry = {
+                        i: seq["SN"] for i, seq in enumerate(bam.header.to_dict()["SQ"])
+                    }
+                    seen = set()
+                    for ref_seq in bam.references:
+                        for hit in bam.fetch(ref_seq):
+                            if (
+                                hit.is_unmapped
+                                or hit.is_secondary
+                                or hit.is_supplementary
+                            ):
+                                continue
+                            key = (hit.reference_id, hit.reference_start)
+                            if key in seen:
+                                continue
+                            seen.add(key)
+                            tsv.write(
+                                f"{entry[hit.reference_id]}\t{hit.query_name}\t{hit.reference_start}\n"
+                            )
+            except ValueError:
+                tsv.write("dummy_reference\tdummy_read\t1000\n")
+
+    def get_start_coordinates(self) -> list[int]:
+        """
+        Get the coordinates of a TSV-file.
+
+        This function opens a TSV-file and saves all starting coordinates in a list.
+
+        Returns:
+            list[int]: The list containing all starting coordinates.
+
+        Raises:
+            ValueError: If no column with starting coordinates is found.
+        """
+        coordinates = []
+        with open(self.tsv, "r", newline="") as f:
+            reader = csv.DictReader(f, delimiter="\t")
+            for row in reader:
+                val = row.get("mapped_starting_coordinate")
+                if val is None:
+                    raise ValueError("Column with starting coordinates not found.")
+                coordinates.append(int(val))
+        return coordinates

xspect/misclassification_detection/point_pattern_analysis.py

@@ -0,0 +1,102 @@
+"""
+Point pattern density analysis tool for the alignment-based misclassification detection.
+
+Notes:
+Developed by Oemer Cetin as part of Bsc thesis (2025), Goethe University Frankfurt am Main.
+(An Integration of Alignment-Free and Alignment-Based Approaches for Bacterial Taxon Assignment)
+"""
+
+import numpy
+
+__author__ = "Cetin, Oemer"
+
+
+class PointPatternAnalysis:
+    """Class for all point pattern density analysis procedures."""
+
+    def __init__(self, points: list[int], length: int):
+        """
+        Initialise the class for point pattern analysis.
+
+        This method sets up the required list with data points (sorted) and the length of the reference genome.
+        All required intensity for the statistics is also calculated.
+
+        Args:
+            points (list): The start coordinates of mapped regions on the genome.
+            length (int): The length of the reference genome.
+        """
+        if len(points) < 2:
+            raise ValueError("Need at least 2 points.")
+        self.sorted_points = numpy.sort(numpy.asarray(points, dtype=float))
+        self.n = len(points)
+        self.length = float(length)
+
+    def ripleys_k(self) -> tuple[bool, float, float]:
+        """
+        Calculates the K-function for the given point distribution.
+
+        This method calculates the K-function to describe the point distribution.
+        The result is than compared with what would be expected under a completely random distribution.
+        (Under complete randomness the K-function result is 2*r)
+
+        Returns:
+            tuple: A tuple containing the information whether points are clustered or not.
+        """
+        r = 0.01 * self.length
+        left = 0
+        right = 0
+        total_neighbors = 0
+
+        for i in range(self.n):
+            while self.sorted_points[i] - self.sorted_points[left] > r:
+                left += 1
+            if right < i:
+                right = i
+            while (
+                right + 1 < self.n
+                and self.sorted_points[right + 1] - self.sorted_points[i] <= r
+            ):
+                right += 1
+            total_neighbors += right - left
+        k = (self.length / (self.n * (self.n - 1))) * total_neighbors
+        return (k > 2 * r), k, 2 * r
+
+    def ripleys_k_edge_corrected(self) -> tuple[bool, float, float]:
+        """
+        Calculates the K-function for the given point distribution with an edge correction factor.
+
+        This method calculates the K-function to describe the point distribution.
+        This time an additional factor is multiplied for each data point to account for edge effects.
+        The result is than compared with what would be expected under a completely random distribution.
+        (Under complete randomness the K-function result is 2*r)
+
+        Returns:
+            tuple: A tuple containing the information whether the points are clustered or not.
+        """
+        r = 0.01 * self.length
+        left = 0
+        right = 0
+        total_weighted = 0
+
+        for i in range(self.n):
+            while self.sorted_points[i] - self.sorted_points[left] > r:
+                left += 1
+            if right < i:
+                right = i
+            while (
+                right + 1 < self.n
+                and self.sorted_points[right + 1] - self.sorted_points[i] <= r
+            ):
+                right += 1
+
+            neighbors = right - left
+            if neighbors > 0:
+                a = max(0, self.sorted_points[i] - r)
+                b = min(self.length, self.sorted_points[i] + r)
+                overlap = b - a
+                weight = (2 * r) / overlap if overlap > 0 else 0
+
+                total_weighted += weight * neighbors
+
+        k = (self.length / (self.n * (self.n - 1))) * total_weighted
+        return (bool(k > 2 * r)), float(k), 2 * r

xspect/misclassification_detection/simulate_reads.py

@@ -0,0 +1,55 @@
+"""
+Read simulation for the alignment-based misclassification detection (Used for testing purposes).
+
+Notes:
+Developed by Oemer Cetin as part of a Bsc thesis at Goethe University Frankfurt am Main (2025).
+(An Integration of Alignment-Free and Alignment-Based Approaches for Bacterial Taxon Assignment)
+"""
+
+import random
+from Bio import SeqIO
+
+__author__ = "Cetin, Oemer"
+
+
+def extract_random_reads(
+    fasta_file, output_fasta, read_length=150, num_reads=1000, seed=42
+) -> None:
+    """
+    Uniformly extracts reads from a genome and writes them to a FASTA-file.
+
+    Args:
+        fasta_file (str): Path to input FASTA file.
+        output_fasta (str): Output FASTA file to write simulated reads.
+        read_length (int): Length of each read to extract.
+        num_reads (int): Total number of reads to extract.
+        seed (int): A seed for reproducibility.
+
+    Raises:
+        ValueError: If the sequences are shorter than the chosen read length.
+    """
+    random.seed(seed)
+    sequences = [
+        record
+        for record in SeqIO.parse(fasta_file, "fasta")
+        if len(record.seq) >= read_length
+    ]
+    if not sequences:
+        raise ValueError("No sequences long enough for the desired read length.")
+
+    # Probability to extract reads from large contigs is higher
+    seq_lengths = [len(rec.seq) for rec in sequences]
+    total_length = sum(seq_lengths)
+    weights = [single_length / total_length for single_length in seq_lengths]
+
+    with open(output_fasta, "w") as o:
+        for i in range(num_reads):
+            # random.choices() provides a list!
+            selected = random.choices(sequences, weights=weights, k=1)[0]
+            seq_length = len(selected.seq)
+            start = random.randint(0, seq_length - read_length)
+            read_seq = selected.seq[start : start + read_length]
+            o.write(
+                f">read_{i}_{selected.id}_{start}-{start + read_length}\n{read_seq}\n"
+            )
+    print("The reads have been simulated successfully.")

xspect/models/probabilistic_filter_model.py

@@ -1,6 +1,7 @@
 """Probabilistic filter model for sequence data"""
 
 import json
+import shutil
 from math import ceil
 from pathlib import Path
 from typing import Any
@@ -9,9 +10,19 @@ from Bio.SeqRecord import SeqRecord
 from Bio import SeqIO
 from slugify import slugify
 import cobs_index as cobs
-from xspect.definitions import fasta_endings, fastq_endings
+from xspect.definitions import (
+    fasta_endings,
+    fastq_endings,
+    get_xspect_misclassification_path,
+)
 from xspect.file_io import get_record_iterator
+from xspect.misclassification_detection.mapping import MappingHandler
 from xspect.models.result import ModelResult
+from collections import defaultdict
+from xspect.ncbi import NCBIHandler
+from xspect.misclassification_detection.point_pattern_analysis import (
+    PointPatternAnalysis,
+)
 
 
 class ProbabilisticFilterModel:
@@ -231,6 +242,7 @@ class ProbabilisticFilterModel:
         filter_ids: list[str] = None,
         step: int = 1,
         display_name: bool = False,
+        validation: bool = False,
     ) -> ModelResult:
         """
         Returns a model result object for the sequence(s) based on the filters in the model
@@ -248,6 +260,7 @@ class ProbabilisticFilterModel:
                 all results are returned.
             step (int): The step size for the k-mer search. Default is 1.
             display_name (bool): Includes a display name for each tax_ID.
+            validation (bool): Sorts out misclassified reads.
 
         Returns:
             ModelResult: An object containing the hits for each sequence, the number of kmers,
@@ -260,7 +273,7 @@ class ProbabilisticFilterModel:
         """
         if isinstance(sequence_input, (SeqRecord)):
             return ProbabilisticFilterModel.predict(
-                self, [sequence_input], filter_ids, step=step, display_name=display_name
+                self, [sequence_input], filter_ids, step, display_name, validation
             )
 
         if self._is_sequence_list(sequence_input) | self._is_sequence_iterator(
@@ -268,13 +281,17 @@ class ProbabilisticFilterModel:
         ):
             hits = {}
             num_kmers = {}
+            if validation and self._is_sequence_iterator(sequence_input):
+                sequence_input = list(sequence_input)
+
             for individual_sequence in sequence_input:
                 individual_hits = self.calculate_hits(
-                    individual_sequence.seq, filter_ids, step=step
+                    individual_sequence.seq, filter_ids, step
                 )
                 num_kmers[individual_sequence.id] = self._count_kmers(
-                    individual_sequence, step=step
+                    individual_sequence, step
                 )
+
                 if display_name:
                     individual_hits.update(
                         {
@@ -285,7 +302,12 @@ class ProbabilisticFilterModel:
                             for key in list(individual_hits.keys())
                         }
                     )
+
                 hits[individual_sequence.id] = individual_hits
+
+            if validation:
+                hits = self.detecting_misclassification(hits, sequence_input)
+
             return ModelResult(self.slug(), hits, num_kmers, sparse_sampling_step=step)
 
         if isinstance(sequence_input, Path):
@@ -294,6 +316,7 @@ class ProbabilisticFilterModel:
                 get_record_iterator(sequence_input),
                 step=step,
                 display_name=display_name,
+                validation=validation,
             )
 
         raise ValueError(
@@ -476,3 +499,98 @@ class ProbabilisticFilterModel:
             sequence_input,
             (SeqIO.FastaIO.FastaIterator, SeqIO.QualityIO.FastqPhredIterator),
         )
+
+    def detecting_misclassification(
+        self,
+        hits: dict[str, dict[str, int]],
+        seq_records: list[SeqRecord],
+        min_reads: int = 10,
+    ) -> dict[str, dict[str, int]]:
+        """
+        Notes:
+        Developed by Oemer Cetin as part of a Bsc thesis at Goethe University Frankfurt am Main (2025).
+        (An Integration of Alignment-Free and Alignment-Based Approaches for Bacterial Taxon Assignment)
+
+        Detects misclassification for short sequences.
+
+        This function is an alignment-based procedure that groups species by highest XspecT scores.
+        Each species group is mapped against the respective reference genome.
+        Start coordinates are extracted and scanned for local clustering.
+        When local clustering is detected, all sequences belonging to the species are sorted out.
+
+        Args:
+            hits (dict): The species annotations from the prediction step.
+            seq_records (list): The provided sequences.
+            min_reads (int): Minimum amount of reads, that species groups should have.
+
+        Returns:
+            dict: hits where misclassified sequences have been sorted out.
+        """
+        rec_by_id = {record.id: record for record in seq_records}
+        grouped: dict[int, list[SeqRecord]] = defaultdict(list)
+        misclassified = {}
+
+        # group by species annotation
+        for record, score_dict in hits.items():
+            if record == "misclassified":
+                continue
+            sorted_hits = sorted(
+                score_dict.items(), key=lambda entry: entry[1], reverse=True
+            )
+            if sorted_hits[0][1] > sorted_hits[1][1]:  # unique highest score
+                highest_tax_id = int(sorted_hits[0][0])  # tax_id
+                if record in rec_by_id:
+                    # groups all reads with the highest score by tax_id
+                    grouped[highest_tax_id].append(rec_by_id[record])
+        filtered_grouped = {
+            tax_id: seq for tax_id, seq in grouped.items() if len(seq) > min_reads
+        }
+        largest_group = max(
+            filtered_grouped,
+            key=lambda tax_id: len(filtered_grouped[tax_id]),
+            default=None,
+        )
+
+        # mapping procedure
+        handler = NCBIHandler()
+        out_dir = get_xspect_misclassification_path()
+        out_dir.mkdir(parents=True, exist_ok=True)
+        for tax_id, reads in filtered_grouped.items():
+            if tax_id == largest_group:
+                continue
+
+            tax_dir = out_dir / str(tax_id)
+            tax_dir.mkdir(parents=True, exist_ok=True)
+            fasta_path = tax_dir / f"{tax_id}.fasta"
+            SeqIO.write(reads, fasta_path, "fasta")
+            reference_path = tax_dir / f"{tax_id}.fna"
+            # download reference once
+            if not (reference_path.exists() and reference_path.stat().st_size > 0):
+                handler.download_reference_genome(tax_id, tax_dir)
+            if not reference_path.exists():
+                shutil.rmtree(tax_dir)
+                continue
+
+            mapping_handler = MappingHandler(str(reference_path), str(fasta_path))
+            mapping_handler.map_reads_onto_reference()
+            mapping_handler.extract_starting_coordinates()
+            genome_length = mapping_handler.get_total_genome_length()
+            start_coordinates = mapping_handler.get_start_coordinates()
+
+            if len(start_coordinates) < min_reads:
+                continue
+
+            # cluster analysis
+            analysis = PointPatternAnalysis(start_coordinates, genome_length)
+            clustered = analysis.ripleys_k_edge_corrected()
+
+            if clustered[0]:  # True or False
+                bucket = misclassified.setdefault(tax_id, {})
+                for read in reads:
+                    data = hits.pop(read.id, None)  # remove false reads from main hits
+                    if data is not None:
+                        bucket[read.id] = data
+
+        if misclassified:
+            hits["misclassified"] = misclassified
+        return hits

xspect/models/probabilistic_filter_svm_model.py

@@ -184,6 +184,7 @@ class ProbabilisticFilterSVMModel(ProbabilisticFilterModel):
         filter_ids: list[str] = None,
         step: int = 1,
         display_name: bool = False,
+        validation: bool = False,
     ) -> ModelResult:
         """
         Predict the labels of the sequences.
@@ -198,28 +199,27 @@ class ProbabilisticFilterSVMModel(ProbabilisticFilterModel):
             filter_ids (list[str], optional): A list of IDs to filter the predictions.
             step (int, optional): Step size for sparse sampling. Defaults to 1.
             display_name (bool): Includes a display name for each tax_ID.
+            validation (bool): Sorts out misclassified reads .
 
         Returns:
             ModelResult: The result of the prediction containing hits, number of kmers, and the
             predicted label.
         """
         # get scores and format them for the SVM
-        res = super().predict(sequence_input, filter_ids, step, display_name)
+        res = super().predict(
+            sequence_input, filter_ids, step, display_name, validation
+        )
         svm_scores = dict(sorted(res.get_scores()["total"].items()))
         svm_scores = [list(svm_scores.values())]
 
         svm = self._get_svm(filter_ids)
-        svm_prediction = str(svm.predict(svm_scores)[0])
-        if display_name:
-            svm_prediction = f"{svm_prediction} -{self.display_names.get(svm_prediction, 'Unknown')}".replace(
-                self.model_display_name, "", 1
-            )
+        res.hits["misclassified"] = res.misclassified
         return ModelResult(
             self.slug(),
             res.hits,
             res.num_kmers,
             sparse_sampling_step=step,
-            prediction=svm_prediction,
+            prediction=str(svm.predict(svm_scores)[0]),
         )
 
     def _get_svm(self, id_keys) -> SVC:

xspect/models/result.py

@@ -40,6 +40,7 @@ class ModelResult:
         self.sparse_sampling_step = sparse_sampling_step
         self.prediction = prediction
         self.input_source = input_source
+        self.misclassified = self.hits.pop("misclassified", None)
 
     def get_scores(self) -> dict:
         """
@@ -165,6 +166,7 @@ class ModelResult:
             "hits": self.hits,
             "scores": self.get_scores(),
             "num_kmers": self.num_kmers,
+            "misclassified": self.misclassified,
             "input_source": self.input_source,
         }
 

xspect/ncbi.py

@@ -1,10 +1,12 @@
 """NCBI handler for the NCBI Datasets API."""
 
+import shutil
 from enum import Enum
 from pathlib import Path
 import time
 from loguru import logger
 import requests
+import zipfile
 
 # pylint: disable=line-too-long
 
@@ -194,8 +196,9 @@ class NCBIHandler:
         assembly_level: AssemblyLevel,
         assembly_source: AssemblySource,
         count: int,
-        min_n50: int = 10000,
-        exclude_atypical: bool = True,
+        min_n50: int,
+        exclude_atypical: bool,
+        allow_inconclusive: bool,
         exclude_paired_reports: bool = True,
         current_version_only: bool = True,
     ) -> list[str]:
@@ -211,11 +214,13 @@ class NCBIHandler:
             assembly_level (AssemblyLevel): The assembly level to get the accessions for.
             assembly_source (AssemblySource): The assembly source to get the accessions for.
             count (int): The number of accessions to get.
-            min_n50 (int, optional): The minimum contig n50 to filter the accessions. Defaults to 10000.
-            exclude_atypical (bool, optional): Whether to exclude atypical accessions. Defaults to True.
+            min_n50 (int): The minimum contig n50 to filter the accessions.
+            exclude_atypical (bool): Whether to exclude atypical accessions.
+            allow_inconclusive (bool): Whether to allow accessions with an inconclusive taxonomy check status.
             exclude_paired_reports (bool, optional): Whether to exclude paired reports. Defaults to True.
             current_version_only (bool, optional): Whether to get only the current version of the accessions. Defaults to True.
 
+
         Returns:
             list[str]: A list containing the accessions.
         """
@@ -240,8 +245,11 @@ class NCBIHandler:
                 report["accession"]
                 for report in response["reports"]
                 if report["assembly_stats"]["contig_n50"] >= min_n50
-                and report["average_nucleotide_identity"]["taxonomy_check_status"]
-                == "OK"
+                and (
+                    allow_inconclusive
+                    or report["average_nucleotide_identity"]["taxonomy_check_status"]
+                    == "OK"
+                )
             ]
         except (IndexError, KeyError, TypeError):
             logger.debug(
@@ -251,7 +259,13 @@ class NCBIHandler:
         return accessions[:count]  # Limit to count
 
     def get_highest_quality_accessions(
-        self, taxon_id: int, assembly_source: AssemblySource, count: int
+        self,
+        taxon_id: int,
+        assembly_source: AssemblySource,
+        count: int,
+        min_n50: int,
+        exclude_atypical: bool,
+        allow_inconclusive: bool,
     ) -> list[str]:
         """
         Get the highest quality accessions for a given taxon id (based on the assembly level).
@@ -263,6 +277,9 @@ class NCBIHandler:
             taxon_id (int): The taxon id to get the accessions for.
             assembly_source (AssemblySource): The assembly source to get the accessions for.
             count (int): The number of accessions to get.
+            min_n50 (int): The minimum contig n50 to filter the accessions.
+            exclude_atypical (bool): Whether to exclude atypical accessions.
+            allow_inconclusive (bool): Whether to allow accessions with an inconclusive taxonomy check status.
 
         Returns:
             list[str]: A list containing the highest quality accessions.
@@ -274,6 +291,9 @@ class NCBIHandler:
                 assembly_level,
                 assembly_source,
                 count,
+                min_n50=min_n50,
+                exclude_atypical=exclude_atypical,
+                allow_inconclusive=allow_inconclusive,
             )
             if len(set(accessions)) >= count:
                 break
@@ -302,3 +322,58 @@ class NCBIHandler:
         with open(output_dir / "ncbi_dataset.zip", "wb") as f:
             for chunk in response.iter_content(chunk_size=8192):
                 f.write(chunk)
+
+    def download_reference_genome(self, taxon_id: int, output_dir: Path) -> Path | None:
+        """
+        Notes:
+        Developed by Oemer Cetin as part of a Bsc thesis at Goethe University Frankfurt am Main (2025).
+        (An Integration of Alignment-Free and Alignment-Based Approaches for Bacterial Taxon Assignment)
+
+        Downloads the reference genome from the RefSeq-DB for a given taxon ID.
+
+        This function queries the NCBI Datasets API for the reference genome and downloads it.
+
+        Args:
+            taxon_id (int): The taxonomy ID of the species.
+            output_dir (Path): Directory where the genome will be saved.
+
+        Returns:
+            Path: Path to the downloaded ZIP file.
+        """
+        accessions = self.get_accessions(
+            taxon_id=taxon_id,
+            assembly_level=AssemblyLevel.REFERENCE,
+            assembly_source=AssemblySource.REFSEQ,
+            count=1,  # only one reference exists
+            min_n50=0,
+            exclude_atypical=True,
+            allow_inconclusive=False,
+        )
+
+        if not accessions:
+            return None
+
+        logger.info(
+            f"Downloading reference genome for taxon {taxon_id}: {accessions[0]}"
+        )
+        self.download_assemblies(accessions, output_dir)
+
+        zip_path = output_dir / "ncbi_dataset.zip"
+
+        fna_file = ""
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            for file in zip_ref.namelist():
+                if file.endswith(".fna"):
+                    extracted_path = zip_ref.extract(file, path=output_dir)
+                    fna_file = output_dir / f"{taxon_id}.fna"
+                    Path(extracted_path).rename(
+                        fna_file
+                    )  # consistent file name (tax_id)
+                    logger.info(f"Extracted reference genome to {fna_file}")
+                    break
+
+        # clean up
+        zip_path.unlink()
+        shutil.rmtree(output_dir / "ncbi_dataset")
+
+        return fna_file

xspect/train.py

@@ -186,6 +186,11 @@ def train_from_ncbi(
     author: str | None = None,
     author_email: str | None = None,
     ncbi_api_key: str | None = None,
+    min_n50: int = 10000,
+    exclude_atypical: bool = True,
+    allow_inconclusive: bool = False,
+    allow_candidatus: bool = False,
+    allow_sp: bool = False,
 ):
     """
     Train a model using NCBI assembly data for a given genus.
@@ -200,6 +205,11 @@ def train_from_ncbi(
         author (str, optional): Author of the model. Defaults to None.
         author_email (str, optional): Author's email. Defaults to None.
         ncbi_api_key (str, optional): NCBI API key for accessing NCBI resources. Defaults to None.
+        min_n50 (int, optional): Minimum N50 value for assemblies. Defaults to 10000.
+        exclude_atypical (bool, optional): Exclude atypical assemblies. Defaults to True.
+        allow_inconclusive (bool, optional): Allow use of accessions with inconclusive taxonomy check status. Defaults to False.
+        allow_candidatus (bool, optional): Allow use of Candidatus species for training. Defaults to False.
+        allow_sp (bool, optional): Allow use of species with "sp." in their names. Defaults to False.
 
     Raises:
         TypeError: If `genus` is not a string.
@@ -221,8 +231,8 @@ def train_from_ncbi(
     filtered_species_ids = [
         tax_id
         for tax_id in species_ids
-        if "candidatus" not in species_names[tax_id].lower()
-        and " sp." not in species_names[tax_id].lower()
+        if (allow_candidatus or "candidatus" not in species_names[tax_id].lower())
+        and (allow_sp or " sp." not in species_names[tax_id].lower())
     ]
     filtered_species_names = {
         str(tax_id): species_names[tax_id] for tax_id in filtered_species_ids
@@ -231,7 +241,12 @@ def train_from_ncbi(
     accessions = {}
     for tax_id in filtered_species_ids:
         taxon_accessions = ncbi_handler.get_highest_quality_accessions(
-            tax_id, AssemblySource.REFSEQ, 8
+            tax_id,
+            AssemblySource.REFSEQ,
+            8,
+            min_n50,
+            exclude_atypical,
+            allow_inconclusive,
         )
         if not taxon_accessions:
             logger.warning(f"No assemblies found for tax_id {tax_id}. Skipping.")
@@ -241,7 +256,9 @@ def train_from_ncbi(
 
     if not accessions:
         raise ValueError(
-            "No species with accessions found. Please check the genus name."
+            "No species with accessions found. "
+            "Please check if the genus name is correct or if there are any data quality issues "
+            "(e. g. inconclusive taxonomy check status, atypical assemblies, low N50 values)."
         )
 
     with TemporaryDirectory() as tmp_dir:

{xspect-0.5.4.dist-info → xspect-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: XspecT
-Version: 0.5.4
+Version: 0.6.0
 Summary: Tool to monitor and characterize pathogens using Bloom filters.
 License: MIT License
         
@@ -45,6 +45,9 @@ Requires-Dist: xxhash
 Requires-Dist: fastapi
 Requires-Dist: uvicorn
 Requires-Dist: python-multipart
+Requires-Dist: mappy
+Requires-Dist: pysam
+Requires-Dist: numpy
 Provides-Extra: docs
 Requires-Dist: mkdocs-material; extra == "docs"
 Requires-Dist: mkdocs-include-markdown-plugin; extra == "docs"

{xspect-0.5.4.dist-info → xspect-0.6.0.dist-info}/RECORD

@@ -1,23 +1,27 @@
 xspect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/classify.py,sha256=SOTfUsEarZVbVtBuwNTlOYQ_MvoW0ADAqH7rTASWEI8,3936
-xspect/definitions.py,sha256=8PpU8bpzcwv8PPacncywz-Na_MicMl-JsvjiX3e46yo,2734
+xspect/classify.py,sha256=fpXk8KobN9QQGSWu5pPR9g65bGxlOmcYEwZ5FJ5KSdM,4106
+xspect/definitions.py,sha256=eP2ttvoW9izCfMYqjLL6Y2oead44l74oHFKRDssTIF8,3506
 xspect/download_models.py,sha256=VALcnowzkUpR-OAvgB5BUdEq9WnyNbli0CxH3OT40Rc,1121
 xspect/file_io.py,sha256=QX2nBtlLAexBdfUr7rtHLlWOuXiaKvfRdpn1Dn0avnY,8120
 xspect/filter_sequences.py,sha256=QKjgUCk3RBY3U9hHmyvSQeQt8n1voBna-NjOoTqdp3A,5196
-xspect/main.py,sha256=vRCsSH_QVKY6usU5d5pjehPAhk4WJfZ_eyl_0xUGu5E,14137
+xspect/main.py,sha256=dhJa6mUneSI0nkwCWHqHrk9sZJgP0MkaPTSceSq1s4c,15463
 xspect/model_management.py,sha256=yWbCk6tUn7-OYpzH0BViX2oWr4cdNkEBjrvnaw5GPdQ,4893
-xspect/ncbi.py,sha256=VRbFvtfGR4WTsc3buZE9UCabE3OJUTRphDRY20g63-E,11704
-xspect/train.py,sha256=jxjK4OqzTywmd5KGPou9A-doH8Nwhlv_xF4X7M6X_jI,11588
+xspect/ncbi.py,sha256=RIwSJcPDREvk_YTNPfT34hQ4mb9nRKoXeNAS8wnLrHY,14413
+xspect/train.py,sha256=VQ5pISDtp0rlGwrYqK_4_OH9CE6n7L1DNZ3txeTty6M,12574
 xspect/web.py,sha256=kM4BZ3fA0f731EEXScAaiGrJZvjjfep1iC1iZemfazw,7039
+xspect/misclassification_detection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xspect/misclassification_detection/mapping.py,sha256=xSibBi7-NoR74jakoYBmTW4Gy5FMfRDVF8H3ef5pElI,6698
+xspect/misclassification_detection/point_pattern_analysis.py,sha256=5ivksPAh0zGMVI33ETPrZ01tPE86ELOwI7-XpJNYsQA,3820
+xspect/misclassification_detection/simulate_reads.py,sha256=fxfKNSAwDT7Vnuh-_vgrMTLrfKihocZPU0gokNicey0,2009
 xspect/mlst_feature/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 xspect/mlst_feature/mlst_helper.py,sha256=pxRX_nRbrTSIFPf_FDV3dxR_FonmGtxttFgqNS7sIxE,8130
 xspect/mlst_feature/pub_mlst_handler.py,sha256=gX0bgAqXTaW9weWgxcbsiD7UtMGuDD9veE9mj42Ffm8,7685
 xspect/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 xspect/models/probabilistic_filter_mlst_model.py,sha256=w9ibUkAYA-DSOEkU8fBenlENrs8JwRRLaF5KO1HVKoM,17716
-xspect/models/probabilistic_filter_model.py,sha256=pUgkN4E2EO-gePVR4BMndgMhJcyvOfVzfjVypjIz2JA,19047
-xspect/models/probabilistic_filter_svm_model.py,sha256=n_HMARvcUMP1i-csiW8uvskcocrvhWMjue7kfsaKPpI,11146
+xspect/models/probabilistic_filter_model.py,sha256=9CyNG-wcvjz1AORzr2hCO_laerYfen1OmWKbRh02nc8,23777
+xspect/models/probabilistic_filter_svm_model.py,sha256=WRXrAWr6f35B4VxgIhhLi1uj8RUZQAAhivjQdePsWhs,11094
 xspect/models/probabilistic_single_filter_model.py,sha256=vJvKZrAybYHq_UdKQ2GvvVwgTYwqRrL-nDDQZxb6RRc,6828
-xspect/models/result.py,sha256=Wpsm9EYrvMazDO0JAqF51Sb8BJqAZwYx4G6-SUOt5-c,7070
+xspect/models/result.py,sha256=v1wslD4RHgoTSwqOVtejfSbEhg4jZ0CDPG55nyhklUg,7185
 xspect/xspect-web/.gitignore,sha256=_nGOe6uxTzy60tl_CIibnOUhXtP-DkOyuM-_s7m4ROg,253
 xspect/xspect-web/README.md,sha256=Fa5cCk66ohbqD_AAVgnXUZLhuzshnLxhlUFhxyscScc,1942
 xspect/xspect-web/components.json,sha256=5emhfq5JRW9J8Zga-1N5jAcj4B-r8VREXnH7Z6tZGNk,425
@@ -78,9 +82,9 @@ xspect/xspect-web/src/components/ui/switch.tsx,sha256=uIqRXtd41ba0eusIEUWVyYZv82
 xspect/xspect-web/src/components/ui/table.tsx,sha256=M2-TIHKwPFWuXrwysSufdQRSMJT-K9jPzGOokfU6PXo,2463
 xspect/xspect-web/src/components/ui/tabs.tsx,sha256=BImHKcdDCtrS3CCV1AGgn8qg0b65RB5P-QdH49IAhx0,1955
 xspect/xspect-web/src/lib/utils.ts,sha256=66ibdQiEHKftZBq1OMLmOKqWma1BkO-O60rc1IQYwLE,165
-xspect-0.5.4.dist-info/licenses/LICENSE,sha256=bhBGDKIRUVwYIHGOGO5hshzuVHyqFJajvSOA3XXOLKI,1094
-xspect-0.5.4.dist-info/METADATA,sha256=T1EVSE_qesDZjlSCaq3xgnUN57n0NIFjOIQCi4swsEo,4569
-xspect-0.5.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-xspect-0.5.4.dist-info/entry_points.txt,sha256=L7qliX3pIuwupQxpuOSsrBJCSHYPOPNEzH8KZKQGGUw,43
-xspect-0.5.4.dist-info/top_level.txt,sha256=hdoa4cnBv6OVzpyhMmyxpJxEydH5n2lDciy8urc1paE,7
-xspect-0.5.4.dist-info/RECORD,,
+xspect-0.6.0.dist-info/licenses/LICENSE,sha256=bhBGDKIRUVwYIHGOGO5hshzuVHyqFJajvSOA3XXOLKI,1094
+xspect-0.6.0.dist-info/METADATA,sha256=_3lkPNVzF3iUOFGFaGRavSPfuJ_wdCQODA0qfU5GXJg,4632
+xspect-0.6.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+xspect-0.6.0.dist-info/entry_points.txt,sha256=L7qliX3pIuwupQxpuOSsrBJCSHYPOPNEzH8KZKQGGUw,43
+xspect-0.6.0.dist-info/top_level.txt,sha256=hdoa4cnBv6OVzpyhMmyxpJxEydH5n2lDciy8urc1paE,7
+xspect-0.6.0.dist-info/RECORD,,