XspecT 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

xspect/models/probabilistic_filter_mlst_model.py

@@ -24,6 +24,7 @@ class ProbabilisticFilterMlstSchemeModel:
         base_path: Path,
         fpr: float = 0.001,
     ) -> None:
+        """Initialise a ProbabilisticFilterMlstSchemeModel object."""
         if k < 1:
             raise ValueError("Invalid k value, must be greater than 0")
         if not isinstance(base_path, Path):
@@ -41,7 +42,12 @@ class ProbabilisticFilterMlstSchemeModel:
         self.indices = []
 
     def to_dict(self) -> dict:
-        """Returns a dictionary representation of the model"""
+        """
+        Returns a dictionary representation of the model.
+
+        Returns:
+            dict: The dictionary containing all metadata of an object.
+        """
         return {
             "k": self.k,
             "model_display_name": self.model_display_name,
@@ -54,14 +60,37 @@ class ProbabilisticFilterMlstSchemeModel:
         }
 
     def get_cobs_index_path(self, scheme: str, locus: str) -> Path:
-        """Returns the path to the cobs index"""
+        """
+        Get the path to the cobs indices.
+
+        This function creates a directory based on the scheme name, if it does not exist.
+        A COBS-Index file is created for every locus in a scheme.
+
+        Args:
+            scheme (str): The name of the scheme.
+            locus (str): The name of the locus.
+
+        Returns:
+            Path: The path to the COBS indices.
+        """
         # To differentiate from genus and species models
         cobs_path = self.base_path / f"{scheme}"
         cobs_path.mkdir(exist_ok=True, parents=True)
         return cobs_path / f"{locus}.cobs_compact"
 
     def fit(self, scheme_path: Path) -> None:
-        """Trains a COBS structure for every locus with all its alleles"""
+        """
+        Trains a COBS structure for every locus with all its alleles.
+
+        This function creates COBS-indices.
+        Many attributes of an object are set in this function.
+
+        Args:
+            scheme_path (Path): The path to the scheme directory with all loci.
+
+        Raises:
+            ValueError: If the scheme alleles have not been downloaded prior.
+        """
         if not scheme_path.exists():
             raise ValueError(
                 "Scheme not found. Please make sure to download the schemes prior!"
@@ -112,7 +141,15 @@ class ProbabilisticFilterMlstSchemeModel:
 
     @staticmethod
     def load(scheme_path: Path) -> "ProbabilisticFilterMlstSchemeModel":
-        """Loads the model from a JSON-file"""
+        """
+        Loads the model from a JSON-file.
+
+        Args:
+            scheme_path (Path): The path of the scheme model.
+
+        Returns:
+            ProbabilisticFilterMlstSchemeModel: A trained model from the disk in JSON format.
+        """
         scheme_name = str(scheme_path).split("/")[-1]
         json_path = scheme_path / f"{scheme_name}.json"
         with open(json_path, "r", encoding="utf-8") as file:
@@ -137,8 +174,30 @@ class ProbabilisticFilterMlstSchemeModel:
                 model.indices.append(cobs_index.Search(str(entry), False))
             return model
 
-    def calculate_hits(self, path: Path, sequence: Seq, step: int = 1) -> list[dict]:
-        """Calculates the hits for a sequence"""
+    def calculate_hits(
+        self, cobs_path: Path, sequence: Seq, step: int = 1
+    ) -> list[dict]:
+        """
+        Calculates the hits for a sequence.
+
+        This function has two ways of identifying strain types.
+        Sequences with a length of up to 10000 base pairs are handled without preprocessing.
+        Sequences with a length >= 10000 base pairs are divided into substrings.
+        The results of each substring are added up to find the strain type.
+
+        Args:
+            cobs_path (Path): The path of the COBS-structure directory.
+            sequence (Seq): The input sequence for classification.
+            step (int, optional): The amount of kmers that are passed; defaults to one.
+
+        Returns:
+            list[dict]: The results of the prediction.
+
+        Raises:
+            ValueError: If the model has not been trained.
+            ValueError: If the sequence is shorter than k.
+            ValueError: If the sequence is not a Seq-object.
+        """
         if not isinstance(sequence, Seq):
             raise ValueError("Invalid sequence, must be a Bio.Seq object")
 
@@ -149,7 +208,7 @@ class ProbabilisticFilterMlstSchemeModel:
             raise ValueError("The Model has not been trained yet")
 
         scheme_path_list = []
-        for entry in sorted(path.iterdir()):
+        for entry in sorted(cobs_path.iterdir()):
             if str(entry).endswith(".json"):
                 continue
             file_name = str(entry).split("/")[-1]  # file_name = locus
@@ -166,11 +225,12 @@ class ProbabilisticFilterMlstSchemeModel:
                 split_sequence = self.sequence_splitter(str(sequence), allele_len)
                 for split in split_sequence:
                     res = index.search(split, step=step)
-                    split_result = self.get_cobs_result(res)
+                    split_result = self.get_cobs_result(res, True)
                     if not split_result:
                         continue
                     cobs_results.append(split_result)
 
+                # add all split results of an Allele id into one
                 all_counts = defaultdict(int)
                 for result in cobs_results:
                     for name, value in result.items():
@@ -179,21 +239,36 @@ class ProbabilisticFilterMlstSchemeModel:
                 sorted_counts = dict(
                     sorted(all_counts.items(), key=lambda item: -item[1])
                 )
-                first_key = next(iter(sorted_counts))
-                highest_result = sorted_counts[first_key]
-                result_dict[scheme_path_list[counter]] = sorted_counts
-                highest_results[scheme_path_list[counter]] = {first_key: highest_result}
+                if not sorted_counts:
+                    result_dict = "A Strain type could not be detected because of no kmer matches!"
+                    highest_results[scheme_path_list[counter]] = {"N/A": 0}
+                else:
+                    first_key = next(iter(sorted_counts))
+                    highest_result = sorted_counts[first_key]
+                    result_dict[scheme_path_list[counter]] = sorted_counts
+                    highest_results[scheme_path_list[counter]] = {
+                        first_key: highest_result
+                    }
                 counter += 1
         else:
             for index in self.indices:
                 res = index.search(
                     str(sequence), step=step
                 )  # COBS can't handle Seq-Objects
-                result_dict[scheme_path_list[counter]] = self.get_cobs_result(res)
-                highest_results[scheme_path_list[counter]] = (
-                    self.get_highest_cobs_result(res)
+                result_dict[scheme_path_list[counter]] = self.get_cobs_result(
+                    res, False
+                )
+                first_key, highest_result = next(
+                    iter(result_dict[scheme_path_list[counter]].items())
                 )
+                highest_results[scheme_path_list[counter]] = {first_key: highest_result}
                 counter += 1
+        # check if the strain type has sufficient amount of kmer hits
+        is_valid = self.has_sufficient_score(highest_results, self.avg_locus_bp_size)
+        if not is_valid:
+            highest_results["Attention:"] = (
+                "This strain type is not reliable due to low kmer hit rates!"
+            )
         return [{"Strain type": highest_results}, {"All results": result_dict}]
 
     def predict(
@@ -208,7 +283,20 @@ class ProbabilisticFilterMlstSchemeModel:
         ),
         step: int = 1,
     ) -> MlstResult:
-        """Returns scores for the sequence(s) based on the filters in the model"""
+        """
+        Get scores for the sequence(s) based on the filters in the model.
+
+        Args:
+            cobs_path (Path): The path of the COBS-structure directory.
+            sequence_input (Seq): The input sequence for classification
+            step (int, optional): The amount of kmers that are passed; defaults to one
+
+        Returns:
+            MlstResult: The results of the prediction.
+
+        Raises:
+            ValueError: If the sequence input is invalid.
+        """
         if isinstance(sequence_input, SeqRecord):
             if sequence_input.id == "<unknown id>":
                 sequence_input.id = "test"
@@ -238,31 +326,48 @@ class ProbabilisticFilterMlstSchemeModel:
             " SeqIO FastaIterator, or a SeqIO FastqPhredIterator"
         )
 
-    def get_highest_cobs_result(self, cobs_result: cobs_index.SearchResult) -> dict:
-        """Returns the first entry in a COBS search result."""
-        # counter = 1
-        # dictio = {}
-        for individual_result in cobs_result:
-            # COBS already sorts the result in descending order
-            # The first doc_name has the highest result which is needed to determine the allele
-            return {individual_result.doc_name: individual_result.score}
-
-    def get_cobs_result(self, cobs_result: cobs_index.SearchResult) -> dict:
-        """Returns all entries in a COBS search result."""
+    def get_cobs_result(
+        self, cobs_result: cobs_index.SearchResult, kmer_threshold: bool
+    ) -> dict:
+        """
+        Get every entry in a COBS search result.
+
+        Args:
+            cobs_result (SearchResult): The result of the prediction.
+            kmer_threshold (bool): Applying a kmer threshold to mitigate false positives
+
+        Returns:
+            dict: A dictionary storing the allele id of locus as key and the score as value.
+        """
         return {
             individual_result.doc_name: individual_result.score
             for individual_result in cobs_result
-            if individual_result.score > 50
+            if not kmer_threshold or individual_result.score > 50
         }
 
     def sequence_splitter(self, input_sequence: str, allele_len: int) -> list[str]:
-        """Returns an equally divided sequence in form of a list."""
+        """
+        Get an equally divided sequence in form of a list.
+
+        This function is splitting very long sequences into substrings.
+        The split is based on sequence and allele length.
+        Measures have been taken to not lose kmers while splitting.
+
+        Args:
+            input_sequence (str): The sequence of interest.
+            allele_len (int): The average length of an allele.
+
+        Returns:
+            list[str]: A list containing all substrings of a sequence greater than 10000 bp.
+
+        Raises:
+            ValueError: If the sequence input is invalid.
+        """
+
         # An input sequence will have 10000 or more base pairs.
         sequence_len = len(input_sequence)
 
-        if sequence_len < 100000:
-            substring_length = allele_len // 10
-        elif 100000 <= sequence_len < 1000000:
+        if sequence_len < 1000000:
             substring_length = allele_len
         elif 1000000 <= sequence_len < 10000000:
             substring_length = allele_len * 10
@@ -285,3 +390,26 @@ class ProbabilisticFilterMlstSchemeModel:
             else:
                 substring_list.append(remaining_substring)
         return substring_list
+
+    def has_sufficient_score(
+        self, highest_results: dict, locus_size: list[int]
+    ) -> bool:
+        """
+        Checks if at least one locus in highest_results has a score >= 0.5 * avg base pair size.
+
+        Args:
+            highest_results (dict): Dict where each key is a locus and each value is the kmer score.
+            locus_size (list[int]): List of average base pair sizes per locus (in directory order).
+
+        Returns:
+            bool: True if any locus score >= 0.5 * its avg base pair size, False otherwise.
+        """
+        for i, (locus, allele_score_dict) in enumerate(highest_results.items()):
+            if not allele_score_dict:
+                continue  # skip empty values
+
+            # Take the score (the only value) from the nested dict
+            score = next(iter(allele_score_dict.values()))
+            if score >= 0.5 * locus_size[i]:
+                return True
+        return False

xspect/models/probabilistic_filter_model.py

@@ -56,6 +56,7 @@ class ProbabilisticFilterModel:
     def to_dict(self) -> dict:
         """Returns a dictionary representation of the model"""
         return {
+            "model_slug": self.slug(),
             "k": self.k,
             "model_display_name": self.model_display_name,
             "author": self.author,

xspect/models/result.py

@@ -58,16 +58,28 @@ class ModelResult:
         return total_hits
 
     def get_filter_mask(self, label: str, filter_threshold: float) -> dict[str, bool]:
-        """Return a mask for filtered subsequences."""
-        if filter_threshold < 0 or filter_threshold > 1:
+        """Return a mask for filtered subsequences.
+
+        The mask is a dictionary with subsequence names as keys and boolean values
+        indicating whether the subsequence is above the filter threshold for the given label.
+        A value of -1 for filter_threshold indicates that the subsequence with the maximum score
+        for the given label should be returned.
+        """
+        if filter_threshold < 0 and not filter_threshold == -1 or filter_threshold > 1:
             raise ValueError("The filter threshold must be between 0 and 1.")
 
         scores = self.get_scores()
         scores.pop("total")
-        return {
-            subsequence: score[label] >= filter_threshold
-            for subsequence, score in scores.items()
-        }
+        if not filter_threshold == -1:
+            return {
+                subsequence: score[label] >= filter_threshold
+                for subsequence, score in scores.items()
+            }
+        else:
+            return {
+                subsequence: score[label] == max(score.values())
+                for subsequence, score in scores.items()
+            }
 
     def get_filtered_subsequence_labels(
         self, label: str, filter_threshold: float = 0.7

xspect/ncbi.py

@@ -2,8 +2,8 @@
 
 from enum import Enum
 from pathlib import Path
-import requests
 import time
+import requests
 
 # pylint: disable=line-too-long
 
@@ -55,14 +55,14 @@ class NCBIHandler:
         elapsed_time = now - self.last_request_time
         if elapsed_time < self.min_interval:
             time.sleep(self.min_interval - elapsed_time)
-        self.last_request_time = now  # Update last request time
+        self.last_request_time = now
 
-    def _make_request(self, endpoint: str, timeout: int = 5) -> dict:
+    def _make_request(self, endpoint: str, timeout: int = 15) -> dict:
         """Make a request to the NCBI Datasets API.
 
         Args:
             endpoint (str): The endpoint to make the request to.
-            timeout (int, optional): The timeout for the request in seconds. Defaults to 5.
+            timeout (int, optional): The timeout for the request in seconds. Defaults to 10.
 
         Returns:
             dict: The response from the API.
@@ -229,7 +229,9 @@ class NCBIHandler:
                 == "OK"
             ]
         except (IndexError, KeyError, TypeError):
-            print(f"Could not get accessions for taxon with ID: {taxon_id}. Skipping.")
+            print(
+                f"Could not get {assembly_level.value} accessions for taxon with ID: {taxon_id}. Skipping."
+            )
             return []
         return accessions[:count]  # Limit to count
 
@@ -255,7 +257,7 @@ class NCBIHandler:
 
         self._enforce_rate_limit()
 
-        response = requests.get(self.base_url + endpoint, stream=True, timeout=5)
+        response = requests.get(self.base_url + endpoint, stream=True, timeout=15)
         if response.status_code != 200:
             response.raise_for_status()
 

xspect/train.py

@@ -243,11 +243,19 @@ def train_from_ncbi(
         cobs_dir.mkdir(parents=True, exist_ok=True)
         svm_dir.mkdir(parents=True, exist_ok=True)
 
-        ncbi_handler.download_assemblies(
-            accessions=sum(accessions.values(), []), output_dir=tmp_dir
-        )
-        extract_zip(tmp_dir, tmp_dir)
-        accession_paths = get_ncbi_dataset_accession_paths(tmp_dir / "ncbi_dataset")
+        # download assemblies
+        all_accessions = sum(accessions.values(), [])
+        batch_size = 100
+        accession_paths = {}
+        for i in range(0, len(all_accessions), batch_size):
+            batch = all_accessions[i : i + batch_size]
+            ncbi_handler.download_assemblies(accessions=batch, output_dir=tmp_dir)
+            extract_zip(
+                tmp_dir / "ncbi_dataset.zip", tmp_dir / f"batch-{i}-{i+batch_size}"
+            )
+            accession_paths.update(
+                get_ncbi_dataset_accession_paths(tmp_dir / f"batch-{i}-{i+batch_size}")
+            )
 
         # select accessions
         cobs_accessions = {}

xspect/web.py

@@ -0,0 +1,173 @@
+"""FastAPI-based web application for XspecT."""
+
+from uuid import uuid4
+import json
+from shutil import copyfileobj
+import importlib.resources as pkg_resources
+from fastapi import APIRouter, FastAPI, HTTPException, UploadFile, BackgroundTasks
+from fastapi.responses import RedirectResponse
+from xspect.definitions import get_xspect_runs_path, get_xspect_upload_path
+from xspect.download_models import download_test_models
+import xspect.model_management as mm
+from xspect.train import train_from_ncbi
+from xspect import classify, filter_sequences
+from fastapi.staticfiles import StaticFiles
+
+app = FastAPI()
+app.mount(
+    "/xspect-web",
+    StaticFiles(directory=str(pkg_resources.files("xspect") / "xspect-web" / "dist")),
+    name="static",
+)
+router = APIRouter()
+
+
+@app.get("/")
+def root():
+    """Root endpoint, forwards to /xspect-web/index.html."""
+    return RedirectResponse(url="/xspect-web/index.html")
+
+
+@router.get("/download-filters")
+def download_filters():
+    """Download filters."""
+    download_test_models("http://assets.adrianromberg.com/xspect-models.zip")
+
+
+@router.get("/classification-result")
+def get_classification_result(uuid: str):
+    """Get classification result."""
+    result_path = get_xspect_runs_path() / f"result_{uuid}.json"
+    if not result_path.exists():
+        raise HTTPException(
+            status_code=404, detail="No result found for the specified uuid."
+        )
+    return json.loads(result_path.read_text())
+
+
+@router.post("/classify")
+def classify_post(
+    classification_type: str,
+    model: str,
+    file: str,
+    background_tasks: BackgroundTasks,
+    step: int = 1,
+):
+    """Classify uploaded sample."""
+    input_path = get_xspect_upload_path() / file
+    if not input_path.exists():
+        raise FileNotFoundError(f"File {input_path} does not exist.")
+
+    uuid = str(uuid4())
+
+    if classification_type == "Genus":
+        background_tasks.add_task(
+            classify.classify_genus,
+            model,
+            input_path,
+            get_xspect_runs_path() / f"result_{uuid}.json",
+            step=step,
+        )
+        return {"message": "Classification started.", "uuid": uuid}
+
+    elif classification_type == "Species":
+        background_tasks.add_task(
+            classify.classify_species,
+            model,
+            input_path,
+            get_xspect_runs_path() / f"result_{uuid}.json",
+            step=step,
+        )
+        return {"message": "Classification started.", "uuid": uuid}
+
+    raise NotImplementedError(
+        f"Classification type {classification_type} is not implemented."
+    )
+
+
+router.post("/filter")
+
+
+def filter_post(
+    filter_type: str,
+    model: str,
+    input_file: str,
+    threshold: float,
+    filter_species: str = None,
+):
+    """Filter sequences."""
+    input_path = get_xspect_upload_path() / input_file
+    output_path = get_xspect_upload_path() / f"filtered_{input_file}"
+
+    if not input_path.exists():
+        raise FileNotFoundError(f"File {input_path} does not exist.")
+
+    if filter_type == "Genus":
+        filter_sequences.filter_genus(model, input_path, output_path, threshold)
+        return {"message": "Genus Filtering started."}
+
+    elif filter_type == "Species":
+        filter_sequences.filter_species(
+            model, filter_species, input_path, output_path, threshold
+        )
+        return {"message": "Species Filtering started."}
+
+    raise NotImplementedError(f"Filter type {filter_type} is not implemented.")
+
+
+@router.post("/train")
+def train(genus: str, background_tasks: BackgroundTasks, svm_steps: int = 1):
+    """Train NCBI model."""
+    background_tasks.add_task(train_from_ncbi, genus, svm_steps)
+
+    return {"message": "Training started."}
+
+
+@router.get("/list-models")
+def list_models():
+    """List available models."""
+    return mm.get_models()
+
+
+@router.get("/model-metadata")
+def get_model_metadata(model_slug: str):
+    """Get metadata of a model."""
+    return mm.get_model_metadata(model_slug)
+
+
+@router.post("/model-metadata")
+def post_model_metadata(model_slug: str, author: str, author_email: str):
+    """Update metadata of a model."""
+    try:
+        mm.update_model_metadata(model_slug, author, author_email)
+    except ValueError as e:
+        return {"error": str(e)}
+    return {"message": "Metadata updated."}
+
+
+@router.post("/model-display-name")
+def post_model_display_name(model_slug: str, filter_id: str, display_name: str):
+    """Update display name of a filter in a model."""
+    try:
+        mm.update_model_display_name(model_slug, filter_id, display_name)
+    except ValueError as e:
+        return {"error": str(e)}
+    return {"message": "Display name updated."}
+
+
+@router.post("/upload-file")
+def upload_file(file: UploadFile):
+    """Upload file to the server."""
+    upload_path = get_xspect_upload_path() / file.filename
+
+    if not upload_path.exists():
+        try:
+            with upload_path.open("wb") as buffer:
+                copyfileobj(file.file, buffer)
+        finally:
+            file.file.close()
+
+    return {"filename": file.filename}
+
+
+app.include_router(router, prefix="/api", tags=["api"])

xspect/xspect-web/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

xspect/xspect-web/README.md

@@ -0,0 +1,54 @@
+# React + TypeScript + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules:
+
+```js
+export default tseslint.config({
+  extends: [
+    // Remove ...tseslint.configs.recommended and replace with this
+    ...tseslint.configs.recommendedTypeChecked,
+    // Alternatively, use this for stricter rules
+    ...tseslint.configs.strictTypeChecked,
+    // Optionally, add this for stylistic rules
+    ...tseslint.configs.stylisticTypeChecked,
+  ],
+  languageOptions: {
+    // other options...
+    parserOptions: {
+      project: ['./tsconfig.node.json', './tsconfig.app.json'],
+      tsconfigRootDir: import.meta.dirname,
+    },
+  },
+})
+```
+
+You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules:
+
+```js
+// eslint.config.js
+import reactX from 'eslint-plugin-react-x'
+import reactDom from 'eslint-plugin-react-dom'
+
+export default tseslint.config({
+  plugins: {
+    // Add the react-x and react-dom plugins
+    'react-x': reactX,
+    'react-dom': reactDom,
+  },
+  rules: {
+    // other rules...
+    // Enable its recommended typescript rules
+    ...reactX.configs['recommended-typescript'].rules,
+    ...reactDom.configs.recommended.rules,
+  },
+})
+```

xspect/xspect-web/components.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "new-york",
+  "rsc": false,
+  "tsx": true,
+  "tailwind": {
+    "config": "",
+    "css": "src/index.css",
+    "baseColor": "neutral",
+    "cssVariables": true,
+    "prefix": ""
+  },
+  "aliases": {
+    "components": "@/components",
+    "utils": "@/lib/utils",
+    "ui": "@/components/ui",
+    "lib": "@/lib",
+    "hooks": "@/hooks"
+  },
+  "iconLibrary": "lucide"
+}