XspecT 0.2.6__tar.gz → 0.2.7__tar.gz

{xspect-0.2.6 → xspect-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: XspecT
-Version: 0.2.6
+Version: 0.2.7
 Summary: Tool to monitor and characterize pathogens using Bloom filters.
 License: MIT License
         

{xspect-0.2.6 → xspect-0.2.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "XspecT"
-version = "0.2.6"
+version = "0.2.7"
 description = "Tool to monitor and characterize pathogens using Bloom filters."
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}

{xspect-0.2.6 → xspect-0.2.7}/src/XspecT.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: XspecT
-Version: 0.2.6
+Version: 0.2.7
 Summary: Tool to monitor and characterize pathogens using Bloom filters.
 License: MIT License
         

{xspect-0.2.6 → xspect-0.2.7}/src/XspecT.egg-info/SOURCES.txt

@@ -71,7 +71,6 @@ src/xspect/models/result.py
 src/xspect/train_filter/__init__.py
 src/xspect/train_filter/create_svm.py
 src/xspect/train_filter/extract_and_concatenate.py
-src/xspect/train_filter/html_scrap.py
 src/xspect/train_filter/ncbi_api/__init__.py
 src/xspect/train_filter/ncbi_api/download_assemblies.py
 src/xspect/train_filter/ncbi_api/ncbi_assembly_metadata.py

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/mlst_feature/mlst_helper.py

@@ -1,4 +1,4 @@
-""" Module for utility functions used in other modules regarding MLST. """
+"""Module for utility functions used in other modules regarding MLST."""
 
 __author__ = "Cetin, Oemer"
 

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/models/result.py

@@ -1,4 +1,4 @@
-""" Module for storing the results of XspecT models. """
+"""Module for storing the results of XspecT models."""
 
 from enum import Enum
 

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/pipeline.py

@@ -1,4 +1,4 @@
-""" Module for defining the Pipeline class. """
+"""Module for defining the Pipeline class."""
 
 import json
 from pathlib import Path

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/run.py

@@ -1,4 +1,4 @@
-""" Module with XspecT global run class, which summarizes individual model results. """
+"""Module with XspecT global run class, which summarizes individual model results."""
 
 import json
 from pathlib import Path

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/train.py

@@ -22,7 +22,6 @@ from xspect.train_filter.ncbi_api import (
 )
 from xspect.train_filter import (
     create_svm,
-    html_scrap,
     extract_and_concatenate,
 )
 
@@ -136,14 +135,10 @@ def train_ncbi(genus: str, svm_step: int = 1):
     children_ids = ncbi_children_tree.NCBIChildrenTree(genus).children_ids()
     species_dict = ncbi_taxon_metadata.NCBITaxonMetadata(children_ids).get_metadata()
 
-    # Get all gcf accessions that have Taxonomy check result OK.
-    logger.info("Checking ANI data for updates")
-    ani_gcf = html_scrap.TaxonomyCheck().ani_gcf()
-
     # Look for up to 8 assembly accessions per species.
     logger.info("Getting assembly metadata")
     all_metadata = ncbi_assembly_metadata.NCBIAssemblyMetadata(
-        all_metadata=species_dict, ani_gcf=ani_gcf, count=8, contig_n50=10000
+        all_metadata=species_dict, count=8, contig_n50=10000
     )
     all_metadata = all_metadata.get_all_metadata()
 

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/train_filter/extract_and_concatenate.py

@@ -1,4 +1,4 @@
-""" Module for extracting and concatenating assemblies. """
+"""Module for extracting and concatenating assemblies."""
 
 __author__ = "Berger, Phillip"
 

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/train_filter/ncbi_api/download_assemblies.py

@@ -23,9 +23,9 @@ def download_assemblies(accessions, dir_name, target_folder, zip_file_name):
     """
 
     path = get_xspect_tmp_path() / dir_name / target_folder / zip_file_name
-    api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v1/genome/accession/{','.join(accessions)}/download"
+    api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v2/genome/accession/{','.join(accessions)}/download"
     parameters = {"include_annotation_type": "GENOME_FASTA", "filename": zip_file_name}
     os.makedirs(os.path.dirname(path), exist_ok=True)
-    genome_download = requests.get(api_url, params=parameters, timeout=20)
+    genome_download = requests.get(api_url, params=parameters, timeout=30)
     with open(path, "wb") as f:
         f.write(genome_download.content)

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/train_filter/ncbi_api/ncbi_assembly_metadata.py

@@ -1,4 +1,4 @@
-""" Collects metadata of assemblies from NCBI API """
+"""Collects metadata of assemblies from NCBI API"""
 
 __author__ = "Berger, Phillip"
 
@@ -14,16 +14,14 @@ class NCBIAssemblyMetadata:
 
     _all_metadata: dict
     _count: int
-    _ani_gcf: list
     _parameters: dict
     _accessions: list[str]
     _contig_n50: int
     _all_metadata_complete: dict
 
-    def __init__(self, all_metadata: dict, ani_gcf: list, count=8, contig_n50=10000):
+    def __init__(self, all_metadata: dict, count=8, contig_n50=10000):
         self._all_metadata = all_metadata
         self._count = count
-        self._ani_gcf = ani_gcf
         self._contig_n50 = contig_n50
 
         self._set_parameters()
@@ -72,7 +70,7 @@ class NCBIAssemblyMetadata:
         }
 
     def _make_request(self, taxon: str):
-        api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v1/genome/taxon/{taxon}"
+        api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v2/genome/taxon/{taxon}/dataset_report"
         accessions = []
         count = 0
         for request_type, parameters in self._parameters.items():
@@ -80,17 +78,19 @@ class NCBIAssemblyMetadata:
             response = raw_response.json()
             if response:
                 try:
-                    assemblies = response["assemblies"]
-                    for assembly in assemblies:
-                        curr_assembly = assembly["assembly"]
-                        curr_accession = curr_assembly["assembly_accession"]
-                        curr_contig_n50 = curr_assembly["contig_n50"]
+                    reports = response["reports"]
+                    for report in reports:
+                        accession = report["accession"]
+                        contig_n50 = report["assembly_stats"]["contig_n50"]
+                        taxonomy_check_status = report["average_nucleotide_identity"][
+                            "taxonomy_check_status"
+                        ]
                         if count < self._count:
                             if (
-                                curr_accession in self._ani_gcf
-                                and curr_contig_n50 > self._contig_n50
+                                taxonomy_check_status == "OK"
+                                and contig_n50 > self._contig_n50
                             ):
-                                accessions.append(curr_accession)
+                                accessions.append(accession)
                                 count += 1
                         else:
                             break

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/train_filter/ncbi_api/ncbi_children_tree.py

@@ -24,7 +24,7 @@ class NCBIChildrenTree:
 
     def _request_tree(self):
         """Make the request for the children tree at the NCBI Datasets API."""
-        api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v1/taxonomy/taxon/{self._taxon}/filtered_subtree"
+        api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v2/taxonomy/taxon/{self._taxon}/filtered_subtree"
         raw_response = requests.get(api_url, timeout=5)
         self._response = raw_response.json()["edges"]
         self._parent_taxon_id = str(self._response["1"]["visible_children"][0])

{xspect-0.2.6 → xspect-0.2.7}/src/xspect/train_filter/ncbi_api/ncbi_taxon_metadata.py

@@ -1,4 +1,4 @@
-""" This module is used to retrieve metadata from the NCBI taxonomy database. """
+"""This module is used to retrieve metadata from the NCBI taxonomy database."""
 
 __author__ = "Berger, Phillip"
 
@@ -21,7 +21,7 @@ class NCBITaxonMetadata:
         self._collect_all_metadata()
 
     def _request_metadata(self):
-        api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v1/taxonomy/taxon/{str(self._taxon)}"
+        api_url = f"https://api.ncbi.nlm.nih.gov/datasets/v2/taxonomy/taxon/{str(self._taxon)}"
         raw_response = requests.get(api_url, timeout=5)
         self._response = raw_response.json()["taxonomy_nodes"]
 

{xspect-0.2.6 → xspect-0.2.7}/tests/test_probabilistic_filter_svm_model.py

@@ -1,4 +1,4 @@
-""" Tests for the ProbabilisticFilterSVMModel class. """
+"""Tests for the ProbabilisticFilterSVMModel class."""
 
 # pylint: disable=redefined-outer-name
 

xspect-0.2.6/src/xspect/train_filter/html_scrap.py

@@ -1,114 +0,0 @@
-""" HTML Scraping for the taxonomy check results from NCBI."""
-
-__author__ = "Berger, Phillip"
-
-import datetime
-import pickle
-import sys
-import time
-import requests
-from loguru import logger
-from xspect.definitions import get_xspect_root_path
-
-
-class TaxonomyCheck:
-    """Class to get the GCFs that passed the taxonomy check from NCBI."""
-
-    _main_url = "https://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/ANI_report_prokaryotes.txt"
-    _test_url = "https://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/"
-    _main_path = get_xspect_root_path() / "taxonomy_check.txt"
-    _test_path = get_xspect_root_path() / "tax_check_date.txt"
-    _new_time: list
-    _tax_check_ok = []
-
-    def __init__(self):
-        old_time = self._get_old_tax_date()
-        self._new_time = self._get_new_tax_date()
-        # Both Dates could be found.
-        # Check if the html file was updated since the last time it was downloaded.
-        # If yes than update the file.
-        if self._new_time and old_time:
-            if self._new_time == old_time:
-                logger.info("File was not updated")
-                self._get_old_file()
-            else:
-                logger.info("Updating file")
-                self._update_tax_check()
-
-        # The old date does not exist.
-        # Get the html file for the taxonomy check results.
-        elif self._new_time and not old_time:
-            logger.info("No file was found. Creating new file")
-            self._update_tax_check()
-
-        elif not self._new_time and old_time:
-            self._get_old_file()
-
-        else:
-            logger.error("Nothing was found")
-            logger.error("Aborting")
-            sys.exit()
-
-    def _get_old_tax_date(self):
-        try:
-            with open(self._test_path, "rb") as f:
-                old_time = pickle.load(f)
-                return old_time
-        except FileNotFoundError:
-            return None
-
-    def _get_new_tax_date(self):
-        raw_response = requests.get(self._test_url, timeout=5)
-        data = raw_response.text.split("\n")
-        for line in data:
-            if "ANI_report_prokaryotes.txt" in line:
-                line_parts = line.split()
-                date_parts = line_parts[-3].split("-")
-                date = datetime.date(
-                    int(date_parts[0]), int(date_parts[1]), int(date_parts[2])
-                )
-                time_parts = line_parts[-2].split(":")
-                combined_time = datetime.time(int(time_parts[0]), int(time_parts[1]))
-                new_time = [date, combined_time]
-
-                return new_time
-
-        return None
-
-    def _update_tax_check(self):
-        raw_response = requests.get(self._main_url, timeout=5)
-        all_tax_checks = raw_response.text.split("\n")[1:-1]
-        self._get_gcf_ok(all_tax_checks)
-        self._save_time()
-        self._save_file()
-
-    def _get_gcf_ok(self, all_tax_checks: list):
-        tax_check_ok = []
-        for line in all_tax_checks:
-            line_parts = line.split("\t")
-            gcf = line_parts[1]
-            tax_check_status = line_parts[-1]
-            if tax_check_status == "OK":
-                tax_check_ok.append(gcf)
-
-        self._tax_check_ok = tax_check_ok
-
-    def _save_time(self):
-        with open(self._test_path, "wb") as f:
-            pickle.dump(self._new_time, f)
-
-    def _save_file(self):
-        with open(self._main_path, "wb") as f:
-            pickle.dump(self._tax_check_ok, f)
-
-    def _get_old_file(self):
-        with open(self._main_path, "rb") as f:
-            self._tax_check_ok = pickle.load(f)
-
-    @staticmethod
-    def _get_current_time():
-        return time.asctime(time.localtime()).split()[3]
-
-    def ani_gcf(self):
-        """Returns ANI GCFs that passed the taxonomy check."""
-        return self._tax_check_ok