XspecT 0.2.5__py3-none-any.whl → 0.2.6__py3-none-any.whl

{XspecT-0.2.5.dist-info → XspecT-0.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: XspecT
-Version: 0.2.5
+Version: 0.2.6
 Summary: Tool to monitor and characterize pathogens using Bloom filters.
 License: MIT License
         
@@ -46,14 +46,14 @@ Requires-Dist: fastapi
 Requires-Dist: uvicorn
 Requires-Dist: python-multipart
 Provides-Extra: docs
-Requires-Dist: sphinx ; extra == 'docs'
-Requires-Dist: furo ; extra == 'docs'
-Requires-Dist: myst-parser ; extra == 'docs'
-Requires-Dist: sphinx-copybutton ; extra == 'docs'
-Requires-Dist: sphinx-autobuild ; extra == 'docs'
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: furo; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
+Requires-Dist: sphinx-copybutton; extra == "docs"
+Requires-Dist: sphinx-autobuild; extra == "docs"
 Provides-Extra: test
-Requires-Dist: pytest ; extra == 'test'
-Requires-Dist: pytest-cov ; extra == 'test'
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
 
 # XspecT - Acinetobacter Species Assignment Tool
 ![Test](https://github.com/bionf/xspect2/actions/workflows/test.yml/badge.svg)
@@ -63,7 +63,7 @@ Requires-Dist: pytest-cov ; extra == 'test'
 <img src="/docs/img/logo.png" height="50%" width="50%">
 
 <!-- start intro -->
-XspecT is a Python-based tool to taxonomically classify sequence-reads (or assembled genomes) on the species and/or sub-type level using [Bloom Filters] and a [Support Vector Machine]. It also identifies existing [blaOxa-genes] and provides a list of relevant research papers for further information.
+XspecT is a Python-based tool to taxonomically classify sequence-reads (or assembled genomes) on the species and/or MLST level using [Bloom Filters] and a [Support Vector Machine].
 <br/><br/>
 
 XspecT utilizes the uniqueness of kmers and compares extracted kmers from the input-data to a reference database. Bloom Filter ensure a fast lookup in this process. For a final prediction the results are classified using a Support Vector Machine. 
@@ -88,14 +88,14 @@ pip install xspect
 Please note that Windows and Alpine Linux is currently not supported.
 
 ## Usage
-### Get the Bloomfilters
-To download basic pre-trained filters, you can use the built-in command:
+### Get the models
+To download basic pre-trained models, you can use the built-in command:
 ```
-xspect download-filters
+xspect download-models
 ```
-Additional species filters can be trained using:
+Additional species models can be trained using:
 ```
-xspect train you-ncbi-genus-name
+xspect train-species you-ncbi-genus-name
 ```
 
 ### How to run the web app
@@ -107,7 +107,7 @@ xspect api
 ### How to use the XspecT command line interface
 Run xspect with the configuration you want to run it with as arguments.
 ```
-xspect classify your-genus path/to/your/input-set
+xspect classify-species your-genus path/to/your/input-set
 ```
 For further instructions on how to use the command line interface, please refer to the [documentation] or execute:
 ```

XspecT-0.2.6.dist-info/RECORD

@@ -0,0 +1,34 @@
+xspect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xspect/definitions.py,sha256=Z6RiCOQnsA_i8DPIq-7SUhrKo7KYf1Azp36UZZpcOX4,1419
+xspect/download_models.py,sha256=lml8pSyM0pF-MxghgSRC9noDT4pkUcBZraaTTImVfbA,739
+xspect/fastapi.py,sha256=FDiGXJmLEeTLD83Hem8yV5aoPJ-GhSG5WmDIQRAA_w4,3257
+xspect/file_io.py,sha256=zKhl6Fd9KZAYiD8YgIyje5TbDYk5lxMp1WUrNkGSBo8,2779
+xspect/main.py,sha256=3HqmnMowjkLNwhaZWtY4aeJCyCyT6h_nZWzYIunHfKg,5325
+xspect/model_management.py,sha256=xF-wjVNJbXYv64RajsIcpLfZUvicDyalJEdSeCx3nQI,3542
+xspect/pipeline.py,sha256=h7duhVZ-hupwO_KQPstzFo8KMfMI2yleb9HmtTiMjic,7219
+xspect/run.py,sha256=OJ7pCFqva3AhIYklKjVnqWGooVRO7S3b56kIAy-xabY,1189
+xspect/train.py,sha256=p_5BPh7XNA7R2h8MwpN0-AwzjbNIxdmeMKztP7RU4g8,9499
+xspect/mlst_feature/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xspect/mlst_feature/mlst_helper.py,sha256=omqKmreah5qaspnJ5qKp_9oZsdHxi0tUJnEzZbpWPEw,5916
+xspect/mlst_feature/pub_mlst_handler.py,sha256=oss3CkJNt6041p3qnMdOfoX8ZgUfpB93CUim-Yakc9A,5031
+xspect/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xspect/models/probabilistic_filter_mlst_model.py,sha256=JMc0yBJPo7J9b-GpvhDmzhwWPIKOwatAq0edDgM72PE,11735
+xspect/models/probabilistic_filter_model.py,sha256=zCn5dcuq5Z4pvmsV9igS0lQ1plUi9-Kky_zRflfrIkI,9659
+xspect/models/probabilistic_filter_svm_model.py,sha256=uabDrF1_CSuIWf9wWyQAkqjAuRUBzEZLkv3J6YHfJsM,5641
+xspect/models/probabilistic_single_filter_model.py,sha256=TdGbQp8ylOif7dD13OSWaS-zFNJo8mXOb6BaQ0mcPdo,3810
+xspect/models/result.py,sha256=RuYqagyG5QbFlW408haXzDbPYJ6yS2flcWcJ8Dy6UjM,4834
+xspect/train_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xspect/train_filter/create_svm.py,sha256=w6gq40yHINVfNzLhJfYFykUaNCwpU9AEDcbkUfis3DY,1504
+xspect/train_filter/extract_and_concatenate.py,sha256=lLrczGgfZi2vAGqxq8fcEmJi5pvqyK33JkB_ZoCNYG8,4840
+xspect/train_filter/html_scrap.py,sha256=76VV_ZbvD2I3IxRb62SiQwRPu2tr4fwn1HkfJQYaosM,3809
+xspect/train_filter/ncbi_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xspect/train_filter/ncbi_api/download_assemblies.py,sha256=MB_mxSjCTL05DqIt1WQem8AGU3PjtJnzPndeI9J-AOI,1285
+xspect/train_filter/ncbi_api/ncbi_assembly_metadata.py,sha256=puzDIws-yyBAEHwSAIYUM7g8FpLFmvOKh5xH1EsY8ZE,3830
+xspect/train_filter/ncbi_api/ncbi_children_tree.py,sha256=_8puOsnsKp5lsMV2gZY1ijkfD_BZKG9eXZCX09qph5E,1819
+xspect/train_filter/ncbi_api/ncbi_taxon_metadata.py,sha256=O6JDXC4E6AYaf7NPnb34eSJyZhMB8r--bjoVF_ZsEdA,1868
+XspecT-0.2.6.dist-info/LICENSE,sha256=bhBGDKIRUVwYIHGOGO5hshzuVHyqFJajvSOA3XXOLKI,1094
+XspecT-0.2.6.dist-info/METADATA,sha256=cO5cpS3zD45dpARRO92XG9bCfCbJ1LE-bAaIxkb0bwQ,4714
+XspecT-0.2.6.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+XspecT-0.2.6.dist-info/entry_points.txt,sha256=L7qliX3pIuwupQxpuOSsrBJCSHYPOPNEzH8KZKQGGUw,43
+XspecT-0.2.6.dist-info/top_level.txt,sha256=hdoa4cnBv6OVzpyhMmyxpJxEydH5n2lDciy8urc1paE,7
+XspecT-0.2.6.dist-info/RECORD,,

{XspecT-0.2.5.dist-info → XspecT-0.2.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.2)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

xspect/definitions.py

@@ -40,3 +40,10 @@ def get_xspect_runs_path():
     runs_path = get_xspect_root_path() / "runs"
     runs_path.mkdir(exist_ok=True, parents=True)
     return runs_path
+
+
+def get_xspect_mlst_path():
+    """Return the path to the XspecT runs directory."""
+    mlst_path = get_xspect_root_path() / "mlst"
+    mlst_path.mkdir(exist_ok=True, parents=True)
+    return mlst_path

xspect/{download_filters.py → download_models.py}

@@ -7,8 +7,8 @@ import requests
 from xspect.definitions import get_xspect_model_path, get_xspect_tmp_path
 
 
-def download_test_filters(url):
-    """Download filters."""
+def download_test_models(url):
+    """Download models."""
 
     download_path = get_xspect_tmp_path() / "models.zip"
     extract_path = get_xspect_tmp_path() / "extracted_models"

xspect/fastapi.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from shutil import copyfileobj
 from fastapi import FastAPI, UploadFile, BackgroundTasks
 from xspect.definitions import get_xspect_runs_path, get_xspect_upload_path
-from xspect.download_filters import download_test_filters
+from xspect.download_models import download_test_models
 import xspect.model_management as mm
 from xspect.models.result import StepType
 from xspect.pipeline import ModelExecution, Pipeline, PipelineStep
@@ -17,7 +17,7 @@ app = FastAPI()
 @app.get("/download-filters")
 def download_filters():
     """Download filters."""
-    download_test_filters("https://xspect2.s3.eu-central-1.amazonaws.com/models.zip")
+    download_test_models("https://xspect2.s3.eu-central-1.amazonaws.com/models.zip")
 
 
 @app.get("/classify")

xspect/main.py

@@ -6,13 +6,23 @@ import uuid
 import click
 import uvicorn
 from xspect import fastapi
-from xspect.download_filters import download_test_filters
+from xspect.download_models import download_test_models
 from xspect.train import train_ncbi
 from xspect.models.result import (
     StepType,
 )
-from xspect.definitions import get_xspect_runs_path, fasta_endings, fastq_endings
+from xspect.definitions import (
+    get_xspect_runs_path,
+    fasta_endings,
+    fastq_endings,
+    get_xspect_model_path,
+)
 from xspect.pipeline import ModelExecution, Pipeline, PipelineStep
+from xspect.mlst_feature.mlst_helper import pick_scheme, pick_scheme_from_models_dir
+from xspect.mlst_feature.pub_mlst_handler import PubMLSTHandler
+from xspect.models.probabilistic_filter_mlst_model import (
+    ProbabilisticFilterMlstSchemeModel,
+)
 
 
 @click.group()
@@ -22,10 +32,10 @@ def cli():
 
 
 @cli.command()
-def download_filters():
-    """Download filters."""
-    click.echo("Downloading filters, this may take a while...")
-    download_test_filters("https://xspect2.s3.eu-central-1.amazonaws.com/models.zip")
+def download_models():
+    """Download models."""
+    click.echo("Downloading models, this may take a while...")
+    download_test_models("https://xspect2.s3.eu-central-1.amazonaws.com/models.zip")
 
 
 @cli.command()
@@ -43,7 +53,7 @@ def download_filters():
     help="Sparse sampling step size (e. g. only every 500th kmer for step=500).",
     default=1,
 )
-def classify(genus, path, meta, step):
+def classify_species(genus, path, meta, step):
     """Classify sample(s) from file or directory PATH."""
     click.echo("Classifying...")
     click.echo(f"Step: {step}")
@@ -105,7 +115,7 @@ def classify(genus, path, meta, step):
     help="SVM Sparse sampling step size (e. g. only every 500th kmer for step=500).",
     default=1,
 )
-def train(genus, bf_assembly_path, svm_assembly_path, svm_step):
+def train_species(genus, bf_assembly_path, svm_assembly_path, svm_step):
     """Train model."""
 
     if bf_assembly_path or svm_assembly_path:
@@ -118,6 +128,49 @@ def train(genus, bf_assembly_path, svm_assembly_path, svm_step):
         raise click.ClickException(str(e)) from e
 
 
+@cli.command()
+@click.option(
+    "-c",
+    "--choose_schemes",
+    is_flag=True,
+    help="Choose your own schemes."
+    "Default setting is Oxford and Pasteur scheme of A.baumannii.",
+)
+def train_mlst(choose_schemes):
+    """Download alleles and train bloom filters."""
+    click.echo("Updating alleles")
+    handler = PubMLSTHandler()
+    handler.download_alleles(choose_schemes)
+    click.echo("Download finished")
+    scheme_path = pick_scheme(handler.get_scheme_paths())
+    species_name = str(scheme_path).split("/")[-2]
+    scheme_name = str(scheme_path).split("/")[-1]
+    model = ProbabilisticFilterMlstSchemeModel(
+        31, f"{species_name}:{scheme_name}", get_xspect_model_path()
+    )
+    click.echo("Creating mlst model")
+    model.fit(scheme_path)
+    model.save()
+    click.echo(f"Saved at {model.cobs_path}")
+
+
+@cli.command()
+@click.option(
+    "-p",
+    "--path",
+    help="Path to FASTA-file for mlst identification.",
+    type=click.Path(exists=True, dir_okay=True, file_okay=True),
+)
+def classify_mlst(path):
+    """MLST classify a sample."""
+    click.echo("Classifying...")
+    path = Path(path)
+    scheme_path = pick_scheme_from_models_dir()
+    model = ProbabilisticFilterMlstSchemeModel.load(scheme_path)
+    model.predict(scheme_path, path).save(model.model_display_name, path)
+    click.echo(f"Run saved at {get_xspect_runs_path()}.")
+
+
 @cli.command()
 def api():
     """Open the XspecT FastAPI."""

xspect/mlst_feature/mlst_helper.py

@@ -0,0 +1,155 @@
+""" Module for utility functions used in other modules regarding MLST. """
+
+__author__ = "Cetin, Oemer"
+
+import requests
+import json
+from io import StringIO
+from pathlib import Path
+from Bio import SeqIO
+from xspect.definitions import get_xspect_model_path, get_xspect_runs_path
+
+
+def create_fasta_files(locus_path: Path, fasta_batch: str):
+    """Create Fasta-Files for every allele of a locus."""
+    # fasta_batch = full string of a fasta file containing every allele sequence of a locus
+    for record in SeqIO.parse(StringIO(fasta_batch), "fasta"):
+        number = record.id.split("_")[-1]  # example id = Oxf_cpn60_263
+        output_fasta_file = locus_path / f"Allele_ID_{number}.fasta"
+        if output_fasta_file.exists():
+            continue  # Ignore existing ones
+        with open(output_fasta_file, "w") as allele:
+            SeqIO.write(record, allele, "fasta")
+
+
+def pick_species_number_from_db(available_species: dict) -> str:
+    """Returns the chosen species from all available ones in the database."""
+    # The "database" string can look like this: pubmlst_abaumannii_seqdef
+    for counter, database in available_species.items():
+        print(str(counter) + ":" + database.split("_")[1])
+    print("\nPick one of the above databases")
+    while True:
+        try:
+            choice = input("Choose a species by selecting the corresponding number:")
+            if int(choice) in available_species.keys():
+                chosen_species = available_species.get(int(choice))
+                return chosen_species
+            else:
+                print(
+                    "Wrong input! Try again with a number that is available in the list above."
+                )
+        except ValueError:
+            print(
+                "Wrong input! Try again with a number that is available in the list above."
+            )
+
+
+def pick_scheme_number_from_db(available_schemes: dict) -> str:
+    """Returns the chosen schemes from all available ones of a species."""
+    # List all available schemes of a species database
+    for counter, scheme in available_schemes.items():
+        print(str(counter) + ":" + scheme[0])
+    print("\nPick any available scheme that is listed for download")
+    while True:
+        try:
+            choice = input("Choose a scheme by selecting the corresponding number:")
+            if int(choice) in available_schemes.keys():
+                chosen_scheme = available_schemes.get(int(choice))[1]
+                return chosen_scheme
+            else:
+                print(
+                    "Wrong input! Try again with a number that is available in the above list."
+                )
+        except ValueError:
+            print(
+                "Wrong input! Try again with a number that is available in the above list."
+            )
+
+
+def scheme_list_to_dict(scheme_list: list[str]):
+    """Converts the scheme list attribute into a dictionary with a number as the key."""
+    return dict(zip(range(1, len(scheme_list) + 1), scheme_list))
+
+
+def pick_scheme_from_models_dir() -> Path:
+    """Returns the chosen scheme from models that have been fitted prior."""
+    schemes = {}
+    counter = 1
+    for entry in sorted((get_xspect_model_path() / "MLST").iterdir()):
+        schemes[counter] = entry
+        counter += 1
+    return pick_scheme(schemes)
+
+
+def pick_scheme(available_schemes: dict) -> Path:
+    """Returns the chosen scheme from the scheme list."""
+    if not available_schemes:
+        raise ValueError("No scheme has been chosen for download yet!")
+
+    if len(available_schemes.items()) == 1:
+        return next(iter(available_schemes.values()))
+
+    # List available schemes
+    for counter, scheme in available_schemes.items():
+        # For Strain Typing with an API-POST Request to the db
+        if str(scheme).startswith("http"):
+            scheme_json = requests.get(scheme).json()
+            print(str(counter) + ":" + scheme_json["description"])
+
+        # To pick a scheme after download for fitting
+        else:
+            print(str(counter) + ":" + str(scheme).split("/")[-1])
+
+    print("\nPick a scheme for strain type prediction")
+    while True:
+        try:
+            choice = input("Choose a scheme by selecting the corresponding number:")
+            if int(choice) in available_schemes.keys():
+                chosen_scheme = available_schemes.get(int(choice))
+                return chosen_scheme
+            else:
+                print(
+                    "Wrong input! Try again with a number that is available in the above list."
+                )
+        except ValueError:
+            print(
+                "Wrong input! Try again with a number that is available in the above list."
+            )
+
+
+class MlstResult:
+    """Class for storing mlst results."""
+
+    def __init__(
+        self,
+        scheme_model: str,
+        steps: int,
+        hits: dict[str, list[dict]],
+    ):
+        self.scheme_model = scheme_model
+        self.steps = steps
+        self.hits = hits
+
+    def get_results(self) -> dict:
+        """Stores the result of a prediction in a dictionary."""
+        results = {seq_id: result for seq_id, result in self.hits.items()}
+        return results
+
+    def to_dict(self) -> dict:
+        """Converts all attributes into one dictionary."""
+        result = {
+            "Scheme": self.scheme_model,
+            "Steps": self.steps,
+            "Results": self.get_results(),
+        }
+        return result
+
+    def save(self, display: str, file_path: Path) -> None:
+        """Saves the result inside the "runs" directory"""
+        file_name = str(file_path).split("/")[-1]
+        json_path = get_xspect_runs_path() / "MLST" / f"{file_name}-{display}.json"
+        json_path.parent.mkdir(exist_ok=True, parents=True)
+        json_object = json.dumps(self.to_dict(), indent=4)
+
+        with open(json_path, "w", encoding="utf-8") as file:
+            file.write(json_object)

xspect/mlst_feature/pub_mlst_handler.py

@@ -0,0 +1,119 @@
+"""Module for connecting with the PubMLST database via API requests and downloading allele files."""
+
+__author__ = "Cetin, Oemer"
+
+import requests
+import json
+from xspect.mlst_feature.mlst_helper import (
+    create_fasta_files,
+    pick_species_number_from_db,
+    pick_scheme_number_from_db,
+    pick_scheme,
+    scheme_list_to_dict,
+)
+from xspect.definitions import get_xspect_mlst_path, get_xspect_upload_path
+
+
+class PubMLSTHandler:
+    """Class for communicating with PubMLST and downloading alleles (FASTA-Format) from all loci."""
+
+    base_url = "http://rest.pubmlst.org/db"
+
+    def __init__(self):
+        # Default values: Oxford (1) and Pasteur (2) schemes of A.baumannii species
+        self.scheme_list = [
+            self.base_url + "/pubmlst_abaumannii_seqdef/schemes/1",
+            self.base_url + "/pubmlst_abaumannii_seqdef/schemes/2",
+        ]
+        self.scheme_paths = []
+
+    def get_scheme_paths(self) -> dict:
+        """Returns the scheme paths in a dictionary"""
+        return scheme_list_to_dict(self.scheme_paths)
+
+    def choose_schemes(self) -> None:
+        """Changes the scheme list attribute to feature other schemes from some species"""
+        available_species = {}
+        available_schemes = {}
+        chosen_schemes = []
+        counter = 1
+        # retrieve all available species
+        species_url = PubMLSTHandler.base_url
+        for species_databases in requests.get(species_url).json():
+            for database in species_databases["databases"]:
+                if database["name"].endswith("seqdef"):
+                    available_species[counter] = database["name"]
+                    counter += 1
+        # pick a species out of the available ones
+        chosen_species = pick_species_number_from_db(available_species)
+
+        counter = 1
+        scheme_url = f"{species_url}/{chosen_species}/schemes"
+        for scheme in requests.get(scheme_url).json()["schemes"]:
+            # scheme["description"] stores the name of a scheme.
+            # scheme["scheme"] stores the URL that is needed for downloading all loci.
+            available_schemes[counter] = [scheme["description"], scheme["scheme"]]
+            counter += 1
+
+        # Selection process of available scheme from a species for download (doubles are caught!)
+        while True:
+            chosen_scheme = pick_scheme_number_from_db(available_schemes)
+            (
+                chosen_schemes.append(chosen_scheme)
+                if chosen_scheme not in chosen_schemes
+                else None
+            )
+            choice = input(
+                "Do you want to pick another scheme to download? (y/n):"
+            ).lower()
+            if choice != "y":
+                break
+        self.scheme_list = chosen_schemes
+
+    def download_alleles(self, choice: False):
+        """Downloads every allele FASTA-file from all loci of the scheme list attribute"""
+        if choice:  # pick an own scheme if not Oxford or Pasteur
+            self.choose_schemes()  # changes the scheme_list attribute
+
+        for scheme in self.scheme_list:
+            scheme_json = requests.get(scheme).json()
+            # We only want the name and the respective featured loci of a scheme
+            scheme_name = scheme_json["description"]
+            locus_list = scheme_json["loci"]
+
+            species_name = scheme.split("_")[1]  # name = pubmlst_abaumannii_seqdef
+            scheme_path = get_xspect_mlst_path() / species_name / scheme_name
+            self.scheme_paths.append(scheme_path)
+
+            for locus_url in locus_list:
+                # After using split the last part ([-1]) of the url is the locus name
+                locus_name = locus_url.split("/")[-1]
+                locus_path = (
+                    get_xspect_mlst_path() / species_name / scheme_name / locus_name
+                )
+
+                if not locus_path.exists():
+                    locus_path.mkdir(exist_ok=True, parents=True)
+
+                alleles = requests.get(f"{locus_url}/alleles_fasta").text
+                create_fasta_files(locus_path, alleles)
+
+    def assign_strain_type_by_db(self):
+        """Sends an API-POST-Request to the database for MLST without bloom filters"""
+        scheme_url = (
+            str(pick_scheme(scheme_list_to_dict(self.scheme_list))) + "/sequence"
+        )
+        fasta_file = get_xspect_upload_path() / "Test.fna"
+        with open(fasta_file, "r") as file:
+            data = file.read()
+            payload = {  # Essential API-POST-Body
+                "sequence": data,
+                "filetype": "fasta",
+            }
+        response = requests.post(scheme_url, data=json.dumps(payload)).json()
+
+        for locus, meta_data in response["exact_matches"].items():
+            # meta_data is a list containing a dictionary, therefore [0] and then key value.
+            # Example: 'Pas_fusA': [{'href': some URL, 'allele_id': '2'}]
+            print(locus + ":" + meta_data[0]["allele_id"], end="; ")
+        print("\nStrain Type:", response["fields"])

xspect/model_management.py

@@ -30,12 +30,11 @@ def get_model_by_slug(model_slug: str):
     model_metadata = get_model_metadata(model_path)
     if model_metadata["model_class"] == "ProbabilisticSingleFilterModel":
         return ProbabilisticSingleFilterModel.load(model_path)
-    elif model_metadata["model_class"] == "ProbabilisticFilterSVMModel":
+    if model_metadata["model_class"] == "ProbabilisticFilterSVMModel":
         return ProbabilisticFilterSVMModel.load(model_path)
-    elif model_metadata["model_class"] == "ProbabilisticFilterModel":
+    if model_metadata["model_class"] == "ProbabilisticFilterModel":
         return ProbabilisticFilterModel.load(model_path)
-    else:
-        raise ValueError(f"Model class {model_metadata['model_class']} not recognized.")
+    raise ValueError(f"Model class {model_metadata['model_class']} not recognized.")
 
 
 def get_model_metadata(model: str | Path):

xspect/models/probabilistic_filter_mlst_model.py

@@ -0,0 +1,287 @@
+"""Probabilistic filter MLST model for sequence data"""
+
+__author__ = "Cetin, Oemer"
+
+import cobs_index
+import json
+from pathlib import Path
+from Bio import SeqIO
+from Bio.Seq import Seq
+from Bio.SeqRecord import SeqRecord
+from cobs_index import DocumentList
+from collections import defaultdict
+from xspect.file_io import get_record_iterator
+from xspect.mlst_feature.mlst_helper import MlstResult
+
+
+class ProbabilisticFilterMlstSchemeModel:
+    """Probabilistic filter MLST scheme model for sequence data"""
+
+    def __init__(
+        self,
+        k: int,
+        model_display_name: str,
+        base_path: Path,
+        fpr: float = 0.001,
+    ) -> None:
+        if k < 1:
+            raise ValueError("Invalid k value, must be greater than 0")
+        if not isinstance(base_path, Path):
+            raise ValueError("Invalid base path, must be a pathlib.Path object")
+
+        self.k = k
+        self.model_display_name = model_display_name
+        self.base_path = base_path / "MLST"
+        self.fpr = fpr
+        self.model_type = "Strain"
+        self.loci = {}
+        self.scheme_path = ""
+        self.cobs_path = ""
+        self.avg_locus_bp_size = []
+        self.indices = []
+
+    def to_dict(self) -> dict:
+        """Returns a dictionary representation of the model"""
+        return {
+            "k": self.k,
+            "model_display_name": self.model_display_name,
+            "model_type": self.model_type,
+            "fpr": self.fpr,
+            "scheme_path": str(self.scheme_path),
+            "cobs_path": str(self.cobs_path),
+            "average_locus_base_pair_size": self.avg_locus_bp_size,
+            "loci": self.loci,
+        }
+
+    def get_cobs_index_path(self, scheme: str, locus: str) -> Path:
+        """Returns the path to the cobs index"""
+        # To differentiate from genus and species models
+        cobs_path = self.base_path / f"{scheme}"
+        cobs_path.mkdir(exist_ok=True, parents=True)
+        return cobs_path / f"{locus}.cobs_compact"
+
+    def fit(self, scheme_path: Path) -> None:
+        """Trains a COBS structure for every locus with all its alleles"""
+        if not scheme_path.exists():
+            raise ValueError(
+                "Scheme not found. Please make sure to download the schemes prior!"
+            )
+
+        scheme = str(scheme_path).split("/")[-1]
+        cobs_path = ""
+        # COBS structure for every locus (default = 7 for Oxford or Pasteur scheme)
+        for locus_path in sorted(scheme_path.iterdir()):
+            locus = str(locus_path).split("/")[-1]
+            # counts all fasta files that belong to a locus
+            self.loci[locus] = sum(
+                (1 for _ in locus_path.iterdir() if not str(_).endswith("cache"))
+            )
+
+            # determine the avg base pair size of alleles
+            fasta_file = next(locus_path.glob("*.fasta"), None)
+            with open(fasta_file, "r") as handle:
+                record = next(SeqIO.parse(handle, "fasta"))
+            self.avg_locus_bp_size.append(len(record.seq))
+
+            # COBS only accepts strings as paths
+            doclist = DocumentList(str(locus_path))
+            index_params = cobs_index.CompactIndexParameters()
+            index_params.term_size = self.k  # k-mer size
+            index_params.clobber = True  # overwrite output and temporary files
+            index_params.false_positive_rate = self.fpr
+
+            # Creates COBS data structure for each locus
+            cobs_path = self.get_cobs_index_path(scheme, locus)
+            cobs_index.compact_construct_list(doclist, str(cobs_path), index_params)
+            # Saves COBS-file inside the "indices" attribute
+            self.indices.append(cobs_index.Search(str(cobs_path)))
+
+        self.scheme_path = scheme_path
+        self.cobs_path = cobs_path.parent
+
+    def save(self) -> None:
+        """Saves the model to disk"""
+        scheme = str(self.scheme_path).split("/")[
+            -1
+        ]  # [-1] -> contains the scheme name
+        json_path = self.base_path / scheme / f"{scheme}.json"
+        json_object = json.dumps(self.to_dict(), indent=4)
+
+        with open(json_path, "w", encoding="utf-8") as file:
+            file.write(json_object)
+
+    @staticmethod
+    def load(scheme_path: Path) -> "ProbabilisticFilterMlstSchemeModel":
+        """Loads the model from a JSON-file"""
+        scheme_name = str(scheme_path).split("/")[-1]
+        json_path = scheme_path / f"{scheme_name}.json"
+        with open(json_path, "r", encoding="utf-8") as file:
+            json_object = file.read()
+            model_json = json.loads(json_object)
+            model = ProbabilisticFilterMlstSchemeModel(
+                model_json["k"],
+                model_json["model_display_name"],
+                json_path.parent,
+                model_json["fpr"],
+            )
+            model.scheme_path = model_json["scheme_path"]
+            model.cobs_path = model_json["cobs_path"]
+            model.avg_locus_bp_size = model_json["average_locus_base_pair_size"]
+            model.loci = model_json["loci"]
+
+            for entry in sorted(json_path.parent.iterdir()):
+                if not entry.exists():
+                    raise FileNotFoundError(f"Index file not found at {entry}")
+                if str(entry).endswith(".json"):  # only COBS-files
+                    continue
+                model.indices.append(cobs_index.Search(str(entry), False))
+            return model
+
+    def calculate_hits(self, path: Path, sequence: Seq, step: int = 1) -> list[dict]:
+        """Calculates the hits for a sequence"""
+        if not isinstance(sequence, Seq):
+            raise ValueError("Invalid sequence, must be a Bio.Seq object")
+
+        if not len(sequence) > self.k:
+            raise ValueError("Invalid sequence, must be longer than k")
+
+        if not self.indices:
+            raise ValueError("The Model has not been trained yet")
+
+        scheme_path_list = []
+        for entry in sorted(path.iterdir()):
+            if str(entry).endswith(".json"):
+                continue
+            file_name = str(entry).split("/")[-1]  # file_name = locus
+            scheme_path_list.append(file_name.split(".")[0])  # without the file ending
+
+        result_dict = {}
+        highest_results = {}
+        counter = 0
+        # split the sequence in parts based on sequence length
+        if len(sequence) >= 10000:
+            for index in self.indices:
+                cobs_results = []
+                allele_len = self.avg_locus_bp_size[counter]
+                split_sequence = self.sequence_splitter(str(sequence), allele_len)
+                for split in split_sequence:
+                    res = index.search(split, step=step)
+                    split_result = self.get_cobs_result(res)
+                    if not split_result:
+                        continue
+                    cobs_results.append(split_result)
+
+                all_counts = defaultdict(int)
+                for result in cobs_results:
+                    for name, value in result.items():
+                        all_counts[name] += value
+
+                sorted_counts = dict(
+                    sorted(all_counts.items(), key=lambda item: -item[1])
+                )
+                first_key = next(iter(sorted_counts))
+                highest_result = sorted_counts[first_key]
+                result_dict[scheme_path_list[counter]] = sorted_counts
+                highest_results[scheme_path_list[counter]] = {first_key: highest_result}
+                counter += 1
+        else:
+            for index in self.indices:
+                res = index.search(
+                    str(sequence), step=step
+                )  # COBS can't handle Seq-Objects
+                result_dict[scheme_path_list[counter]] = self.get_cobs_result(res)
+                highest_results[scheme_path_list[counter]] = (
+                    self.get_highest_cobs_result(res)
+                )
+                counter += 1
+        return [{"Strain type": highest_results}, {"All results": result_dict}]
+
+    def predict(
+        self,
+        cobs_path: Path,
+        sequence_input: (
+            SeqRecord
+            | list[SeqRecord]
+            | SeqIO.FastaIO.FastaIterator
+            | SeqIO.QualityIO.FastqPhredIterator
+            | Path
+        ),
+        step: int = 1,
+    ) -> MlstResult:
+        """Returns scores for the sequence(s) based on the filters in the model"""
+        if isinstance(sequence_input, SeqRecord):
+            if sequence_input.id == "<unknown id>":
+                sequence_input.id = "test"
+            hits = {
+                sequence_input.id: self.calculate_hits(cobs_path, sequence_input.seq)
+            }
+            return MlstResult(self.model_display_name, step, hits)
+
+        if isinstance(sequence_input, Path):
+            return ProbabilisticFilterMlstSchemeModel.predict(
+                self, cobs_path, get_record_iterator(sequence_input), step=step
+            )
+
+        if isinstance(
+            sequence_input,
+            (SeqIO.FastaIO.FastaIterator, SeqIO.QualityIO.FastqPhredIterator),
+        ):
+            hits = {}
+            # individual_seq is a SeqRecord-Object
+            for individual_seq in sequence_input:
+                individual_hits = self.calculate_hits(cobs_path, individual_seq.seq)
+                hits[individual_seq.id] = individual_hits
+            return MlstResult(self.model_display_name, step, hits)
+
+        raise ValueError(
+            "Invalid sequence input, must be a Seq object, a list of Seq objects, a"
+            " SeqIO FastaIterator, or a SeqIO FastqPhredIterator"
+        )
+
+    def get_highest_cobs_result(self, cobs_result: cobs_index.SearchResult) -> dict:
+        """Returns the first entry in a COBS search result."""
+        # counter = 1
+        # dictio = {}
+        for individual_result in cobs_result:
+            # COBS already sorts the result in descending order
+            # The first doc_name has the highest result which is needed to determine the allele
+            return {individual_result.doc_name: individual_result.score}
+
+    def get_cobs_result(self, cobs_result: cobs_index.SearchResult) -> dict:
+        """Returns all entries in a COBS search result."""
+        return {
+            individual_result.doc_name: individual_result.score
+            for individual_result in cobs_result
+            if individual_result.score > 50
+        }
+
+    def sequence_splitter(self, input_sequence: str, allele_len: int) -> list[str]:
+        """Returns an equally divided sequence in form of a list."""
+        # An input sequence will have 10000 or more base pairs.
+        sequence_len = len(input_sequence)
+
+        if sequence_len < 100000:
+            substring_length = allele_len // 10
+        elif 100000 <= sequence_len < 1000000:
+            substring_length = allele_len
+        elif 1000000 <= sequence_len < 10000000:
+            substring_length = allele_len * 10
+        else:
+            substring_length = allele_len * 100
+
+        substring_list = []
+        start = 0
+
+        while start + substring_length <= sequence_len:
+            substring_list.append(input_sequence[start : start + substring_length])
+            start += substring_length - self.k + 1  # To not lose kmers when dividing
+
+        # The remaining string is either appended to the list or added to the last entry.
+        if start < len(input_sequence):
+            remaining_substring = input_sequence[start:]
+            # A substring needs to be at least of size k for COBS.
+            if len(remaining_substring) < self.k:
+                substring_list[-1] += remaining_substring
+            else:
+                substring_list.append(remaining_substring)
+        return substring_list

xspect/models/probabilistic_filter_model.py

@@ -8,6 +8,7 @@ from Bio.SeqRecord import SeqRecord
 from Bio import SeqIO
 from slugify import slugify
 import cobs_index as cobs
+from xspect.definitions import fasta_endings, fastq_endings
 from xspect.file_io import get_record_iterator
 from xspect.models.result import ModelResult
 
@@ -64,10 +65,6 @@ class ProbabilisticFilterModel:
             "num_hashes": self.num_hashes,
         }
 
-    def __dict__(self) -> dict:
-        """Returns a dictionary representation of the model"""
-        return self.to_dict()
-
     def slug(self) -> str:
         """Returns a slug representation of the model"""
         return slugify(self.model_display_name + "-" + str(self.model_type))
@@ -89,13 +86,7 @@ class ProbabilisticFilterModel:
 
         doclist = cobs.DocumentList()
         for file in dir_path.iterdir():
-            if file.is_file() and file.suffix in [
-                ".fasta",
-                ".fna",
-                ".fa",
-                ".fastq",
-                ".fq",
-            ]:
+            if file.is_file() and file.suffix[1:] in fasta_endings + fastq_endings:
                 # cobs only uses the file name to the first "." as the document name
                 if file.name in display_names:
                     self.display_names[file.name.split(".")[0]] = display_names[

xspect/models/probabilistic_filter_svm_model.py

@@ -65,8 +65,11 @@ class ProbabilisticFilterSVMModel(ProbabilisticFilterModel):
     ) -> None:
         """Fit the SVM to the sequences and labels"""
 
+        # Since the SVM works with score data, we need to train
+        # the underlying data structure for score generation first
         super().fit(dir_path, display_names=display_names)
 
+        # calculate scores for SVM training
         score_list = []
         for file in svm_path.iterdir():
             if not file.is_file():

xspect/models/probabilistic_single_filter_model.py

@@ -1,4 +1,4 @@
-"""Probabilistic filter SVM model for sequence data"""
+"""Base probabilistic filter model for sequence data"""
 
 # pylint: disable=no-name-in-module, too-many-instance-attributes
 
@@ -14,7 +14,7 @@ from xspect.file_io import get_record_iterator
 
 
 class ProbabilisticSingleFilterModel(ProbabilisticFilterModel):
-    """Probabilistic filter SVM model for sequence data"""
+    """Base probabilistic filter model for sequence data"""
 
     def __init__(
         self,
@@ -25,7 +25,6 @@ class ProbabilisticSingleFilterModel(ProbabilisticFilterModel):
         model_type: str,
         base_path: Path,
         fpr: float = 0.01,
-        num_hashes: int = 7,
     ) -> None:
         super().__init__(
             k=k,
@@ -35,12 +34,12 @@ class ProbabilisticSingleFilterModel(ProbabilisticFilterModel):
             model_type=model_type,
             base_path=base_path,
             fpr=fpr,
-            num_hashes=num_hashes,
+            num_hashes=1,
         )
         self.bf = None
 
     def fit(self, file_path: Path, display_name: str) -> None:
-        """Fit the SVM to the sequences and labels"""
+        """Fit the cobs classic index to the sequences and labels"""
         # estimate number of kmers
         total_length = 0
         for record in get_record_iterator(file_path):
@@ -89,7 +88,6 @@ class ProbabilisticSingleFilterModel(ProbabilisticFilterModel):
                 model_json["model_type"],
                 path.parent,
                 fpr=model_json["fpr"],
-                num_hashes=model_json["num_hashes"],
             )
             model.display_names = model_json["display_names"]
             bloom_path = model.base_path / model.slug() / "filter.bloom"

xspect/models/result.py

@@ -5,10 +5,11 @@ from enum import Enum
 
 def get_last_processing_step(result: "ModelResult") -> "ModelResult":
     """Get the last subprocessing step of the result. First path only."""
-    last_step = result
-    while last_step.subprocessing_steps:
-        last_step = last_step.subprocessing_steps[-1].result
-    return last_step
+
+    # traverse result tree to get last step
+    while result.subprocessing_steps:
+        result = result.subprocessing_steps[-1].result
+    return result
 
 
 class StepType(Enum):
@@ -82,9 +83,9 @@ class ModelResult:
         scores = {
             subsequence: {
                 label: round(hits / self.num_kmers[subsequence], 2)
-                for label, hits in subseuqence_hits.items()
+                for label, hits in subsequence_hits.items()
             }
-            for subsequence, subseuqence_hits in self.hits.items()
+            for subsequence, subsequence_hits in self.hits.items()
         }
 
         # calculate total scores

xspect/train.py

@@ -40,7 +40,7 @@ def check_user_input(user_input: str):
         rank = metadata["rank"]
         lineage = metadata["lineage"]
         bacteria_id = 2
-        if not sci_name == user_input and not tax_id == user_input:
+        if user_input not in (sci_name, tax_id):
             print(
                 f"{get_current_time()}| The given genus: {user_input} was found as"
                 f" genus: {sci_name} ID: {tax_id}"
@@ -60,38 +60,6 @@ def check_user_input(user_input: str):
         sys.exit()
 
 
-def copy_custom_data(bf_path: str, svm_path: str, dir_name: str):
-    """
-
-    :param bf_path:
-    :param svm_path:
-    :param dir_name:
-    :return:
-    """
-    path = Path(os.getcwd()) / "genus_metadata" / dir_name
-    new_bf_path = path / "concatenate"
-    new_svm_path = path / "training_data"
-
-    # Make the new directories.
-    path.mkdir(exist_ok=True)
-    new_bf_path.mkdir(exist_ok=True)
-    new_svm_path.mkdir(exist_ok=True)
-
-    # Move bloomfilter files.
-    bf_files = os.listdir(bf_path)
-    for file in bf_files:
-        file_path = Path(bf_path) / file
-        new_file_path = new_bf_path / file
-        shutil.copy2(file_path, new_file_path)
-
-    # Move svm files.
-    svm_files = os.listdir(svm_path)
-    for file in svm_files:
-        file_path = Path(svm_path) / file
-        new_file_path = new_svm_path / file
-        shutil.copy2(file_path, new_file_path)
-
-
 def set_logger(dir_name: str):
     """Sets the logger parameters.
 

XspecT-0.2.5.dist-info/RECORD

@@ -1,30 +0,0 @@
-xspect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/definitions.py,sha256=gg6NvT8ypNzlnJvMMo3nHsyh8DHFFu41lOfnILkRDpE,1215
-xspect/download_filters.py,sha256=ByE7Oggx-AyJ02Wirk_wcJHNdRDrJMfjwhmUe5tgWbE,741
-xspect/fastapi.py,sha256=C8pBBiqM6UdedLZgzfL_YYRuy98aPj8dcw_CLFrtMMc,3260
-xspect/file_io.py,sha256=zKhl6Fd9KZAYiD8YgIyje5TbDYk5lxMp1WUrNkGSBo8,2779
-xspect/main.py,sha256=eOA9PAeq3VvPWWoOZxXFErvPNW-ANzOxqMsbQJPCvDw,3651
-xspect/model_management.py,sha256=w0aqjLUoixCokyKTYrcN1vih5IoLYLJG9p8aeYdVc8Y,3560
-xspect/pipeline.py,sha256=h7duhVZ-hupwO_KQPstzFo8KMfMI2yleb9HmtTiMjic,7219
-xspect/run.py,sha256=OJ7pCFqva3AhIYklKjVnqWGooVRO7S3b56kIAy-xabY,1189
-xspect/train.py,sha256=khC1lldqfr4NvzLUiSJjSlh7DBG1ePielvQMiB29Hl8,10399
-xspect/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/models/probabilistic_filter_model.py,sha256=ImyNRzR7jf2CBPGI65ItG0_eYmrQjo9soQYlsM0r-P0,9829
-xspect/models/probabilistic_filter_svm_model.py,sha256=Z_aAigE_fC_gm80hRfxvROHGs6LuBqZnATHPpAkQGQE,5466
-xspect/models/probabilistic_single_filter_model.py,sha256=nDAd_-_Ci2eH0KOJtf4wA-w63FMq9rGSR1LGiIA-gdw,3884
-xspect/models/result.py,sha256=vHUEFXvbFyB8WmasXp99IrztjwaxH1f9QMFiRUPe40Q,4824
-xspect/train_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/train_filter/create_svm.py,sha256=w6gq40yHINVfNzLhJfYFykUaNCwpU9AEDcbkUfis3DY,1504
-xspect/train_filter/extract_and_concatenate.py,sha256=lLrczGgfZi2vAGqxq8fcEmJi5pvqyK33JkB_ZoCNYG8,4840
-xspect/train_filter/html_scrap.py,sha256=76VV_ZbvD2I3IxRb62SiQwRPu2tr4fwn1HkfJQYaosM,3809
-xspect/train_filter/ncbi_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xspect/train_filter/ncbi_api/download_assemblies.py,sha256=MB_mxSjCTL05DqIt1WQem8AGU3PjtJnzPndeI9J-AOI,1285
-xspect/train_filter/ncbi_api/ncbi_assembly_metadata.py,sha256=puzDIws-yyBAEHwSAIYUM7g8FpLFmvOKh5xH1EsY8ZE,3830
-xspect/train_filter/ncbi_api/ncbi_children_tree.py,sha256=_8puOsnsKp5lsMV2gZY1ijkfD_BZKG9eXZCX09qph5E,1819
-xspect/train_filter/ncbi_api/ncbi_taxon_metadata.py,sha256=O6JDXC4E6AYaf7NPnb34eSJyZhMB8r--bjoVF_ZsEdA,1868
-XspecT-0.2.5.dist-info/LICENSE,sha256=bhBGDKIRUVwYIHGOGO5hshzuVHyqFJajvSOA3XXOLKI,1094
-XspecT-0.2.5.dist-info/METADATA,sha256=NDw2i1MawAjAkybDXzaQfIIGFI4sw86MSlQJ8z6vkWs,4834
-XspecT-0.2.5.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-XspecT-0.2.5.dist-info/entry_points.txt,sha256=L7qliX3pIuwupQxpuOSsrBJCSHYPOPNEzH8KZKQGGUw,43
-XspecT-0.2.5.dist-info/top_level.txt,sha256=hdoa4cnBv6OVzpyhMmyxpJxEydH5n2lDciy8urc1paE,7
-XspecT-0.2.5.dist-info/RECORD,,