XspecT 0.6.0__tar.gz → 0.7.1__tar.gz

{xspect-0.6.0 → xspect-0.7.1}/.github/workflows/docs.yml

@@ -16,7 +16,7 @@ jobs:
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.x
+          python-version: 3.13
       - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
       - uses: actions/cache@v4
         with:

{xspect-0.6.0 → xspect-0.7.1}/.github/workflows/pypi.yml

@@ -25,7 +25,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.x"
+          python-version: "3.13"
 
       - name: Build release distributions
         run: |

{xspect-0.6.0 → xspect-0.7.1}/.github/workflows/test.yml

@@ -19,15 +19,14 @@ jobs:
         - name: Set up Python
           uses: actions/setup-python@v4
           with:
-            python-version: "3.x"
+            python-version: "3.13"
         - name: Install package
           run: |
             python -m pip install --upgrade pip
             pip install '.[test]'
-        - name: Download models and train MLST
+        - name: Download models
           run: |
             xspect models download
-            yes 1 | xspect models train mlst
         - name: Test with pytest
           env:
             NCBI_API_KEY: ${{ secrets.NCBI_API_KEY }}

{xspect-0.6.0 → xspect-0.7.1}/.gitignore

@@ -187,4 +187,8 @@ data/
 results/
 
 # Slurm
-slurm-*
+slurm-*
+delete.slurm
+
+
+playground.ipynb

{xspect-0.6.0 → xspect-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: XspecT
-Version: 0.6.0
+Version: 0.7.1
 Summary: Tool to monitor and characterize pathogens using Bloom filters.
 License: MIT License
         
@@ -29,7 +29,7 @@ Project-URL: Repository, https://github.com/BIONF/XspecT2.git
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Classifier: License :: OSI Approved :: MIT License
-Requires-Python: >=3.10
+Requires-Python: <3.14,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: biopython
@@ -64,19 +64,19 @@ Dynamic: license-file
 [![linting: pylint](https://img.shields.io/badge/linting-pylint-yellowgreen)](https://github.com/pylint-dev/pylint)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
-XspecT is a Python-based tool to taxonomically classify sequence-reads (or assembled genomes) on the species and/or MLST level using [kmer indices] and a [Support Vector Machine].
+XspecT is a Python-based tool to taxonomically classify sequence-reads (or assembled genomes) on the species and/or MLST level using [kmer indices] and a [support vector machine].
 
-XspecT utilizes the uniqueness of kmers and compares extracted kmers from the input-data to a kmer index. Probablistic data structures ensure a fast lookup in this process. For a final prediction, the results are classified using a Support Vector Machine.
+XspecT utilizes the uniqueness of kmers and compares extracted kmers from the input-data to a kmer index. Probablistic data structures ensure a fast lookup in this process. For a final prediction, the results are classified using a support vector machine.
 
 The tool is available as a web-based application and as a command line interface.
 
 [kmer indices]: https://arxiv.org/abs/1905.09624
-[Support Vector Machine]: https://en.wikipedia.org/wiki/Support-vector_machine
+[support vector machine]: https://en.wikipedia.org/wiki/Support-vector_machine
 <!-- end intro -->
 
 <!-- start quickstart -->
 ## Installation
-To install XspecT, please download the lastest 64 bit Python version and install the package using pip:
+To install XspecT, please download Python 3.10 - 3.13 and install the package using pip:
 ```
 pip install xspect
 ```
@@ -114,5 +114,5 @@ For further instructions on how to use the command line interface, please refer
 ```
 xspect --help
 ```
-[documentation]: https://bionf.github.io/XspecT2/cli/index.html
+[documentation]: https://bionf.github.io/XspecT/cli/index.html
 <!-- end quickstart -->

{xspect-0.6.0 → xspect-0.7.1}/README.md

@@ -4,19 +4,19 @@
 [![linting: pylint](https://img.shields.io/badge/linting-pylint-yellowgreen)](https://github.com/pylint-dev/pylint)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
-XspecT is a Python-based tool to taxonomically classify sequence-reads (or assembled genomes) on the species and/or MLST level using [kmer indices] and a [Support Vector Machine].
+XspecT is a Python-based tool to taxonomically classify sequence-reads (or assembled genomes) on the species and/or MLST level using [kmer indices] and a [support vector machine].
 
-XspecT utilizes the uniqueness of kmers and compares extracted kmers from the input-data to a kmer index. Probablistic data structures ensure a fast lookup in this process. For a final prediction, the results are classified using a Support Vector Machine.
+XspecT utilizes the uniqueness of kmers and compares extracted kmers from the input-data to a kmer index. Probablistic data structures ensure a fast lookup in this process. For a final prediction, the results are classified using a support vector machine.
 
 The tool is available as a web-based application and as a command line interface.
 
 [kmer indices]: https://arxiv.org/abs/1905.09624
-[Support Vector Machine]: https://en.wikipedia.org/wiki/Support-vector_machine
+[support vector machine]: https://en.wikipedia.org/wiki/Support-vector_machine
 <!-- end intro -->
 
 <!-- start quickstart -->
 ## Installation
-To install XspecT, please download the lastest 64 bit Python version and install the package using pip:
+To install XspecT, please download Python 3.10 - 3.13 and install the package using pip:
 ```
 pip install xspect
 ```
@@ -54,5 +54,5 @@ For further instructions on how to use the command line interface, please refer
 ```
 xspect --help
 ```
-[documentation]: https://bionf.github.io/XspecT2/cli/index.html
+[documentation]: https://bionf.github.io/XspecT/cli/index.html
 <!-- end quickstart -->

{xspect-0.6.0 → xspect-0.7.1}/docs/benchmark.md

@@ -2,16 +2,16 @@
 
 XspecT is a tool designed for fast and accurate species classification of genome assemblies and simulated reads. To evaluate its classification accuracy, we conducted a benchmark using a set of Acinetobacter genomes.
 
-The benchmark was performed by first download all available Acinetobacter genomes from Genbank, filtered on a passed ("OK") taxonomy check status. Genomes assigned to strain IDs were remapped to their respective species IDs, after which genomes with species IDs not contained in XspecT's Acinetobacter model were removed. The remaining genomes were then used to classify both assemblies and simulated reads generated from them. Simulated reads were generated by first filtering on genomes that were not part of the training data and that were categorized as "complete" by NCBI. The reads were then simulated from the longest contig of each genome (assumed to be the chromosome) using a custom Python script. Up to three genomes were selected per species. 100 000 reads were simulated for each genome, with a read length of 100 bp and no simulated sequencing errors. The reads were then classified using XspecT with predictions based on the maximum-scoring species.
+The benchmark was performed by first downloading all available Acinetobacter genomes from RefSeq, filtered on a passed ("OK") taxonomy check status and on them not being part of the training dataset. Genomes assigned to strain IDs were remapped to their respective species IDs, after which genomes with species IDs not contained in XspecT's Acinetobacter model were removed. The remaining genomes were then used to classify both assemblies and simulated reads generated from them. Simulated reads were generated by first filtering on genomes that were categorized as "complete" or "chromosome" by NCBI. The reads were then simulated from the longest contig of each genome (assumed to be the chromosome) using ART. 100 000 reads were simulated for each genome based on the HiSeq 2500 profile, with a read length of 125 bp. The reads were then classified using XspecT with predictions based on the maximum-scoring species.
 
 ## Benchmark Results
 
-The benchmark results show that XspecT achieves high classification accuracy, with an overall accuracy of nearly 100% for whole genomes and 82% for simulated reads. However, the low macro-average F1 score (0.41) for the read dataset highlights a substantial class imbalance.
+The benchmark results show that XspecT achieves very high classification accuracy of nearly 100% for whole genomes and strong but reduced accuracy of 70% for simulated reads. However, the low macro-average F1 score (0.21) for the read dataset highlights a substantial class imbalance.
 
 | Dataset   | Total Samples | Matches   | Mismatches | Match Rate | Mismatch Rate | Accuracy | Macro Avg F1 | Weighted Avg F1 |
 |-----------|--------------:|----------:|-----------:|-----------:|--------------:|---------:|-------------:|----------------:|
-| Assembly  | 44,905        | 44,879    | 26         | 99.94%     | 0.06%         | ≈1.00    | 0.95         | 1.00            |
-| Reads     | 9,200,000     | 7,526,902 | 1,673,098  | 81.81%     | 18.19%        | 0.82     | 0.41         | 0.87            |
+| Assembly  | 13,795        | 13,776    | 19         | 99.86%     | 0.14%         | ≈1.00    | 0.96         | ≈1.00           |
+| Reads     | 121,590,139   | 85,679,572| 35,910,567 | 70.47%     | 29.53%        | 0.70     | 0.21         | 0.79            |
 
 ## Running the benchmark yourself
 

{xspect-0.6.0 → xspect-0.7.1}/docs/cli.md

@@ -27,7 +27,7 @@ This will show a list of all available models, separated by their type (species,
 
 ### Downloading Models
 
-To download a basic set of pre-trained models (Acinetobacter and Salonella), run:
+To download a basic set of pre-trained models (Acinetobacter, including Oxford MLST scheme, and Salonella), run:
 
 ```bash
 xspect models download
@@ -43,7 +43,7 @@ To train a model with NCBI data, run the following command:
 xspect models train ncbi
 ```
 
-By default, XspecT filters out NCBI accessions that do not meed minimum N50 thresholds, have an inconclusive taxonomy check status, or are deemed atypical by NCBI. Furthermore, species with "Candidatus" and "sp." in their species names are filtered out. To disable filtering behavior, use the respective flag (see `xspect models train ncbi --help`).
+By default, XspecT filters out NCBI accessions that do not meet minimum N50 thresholds, have an inconclusive taxonomy check status, or are deemed atypical by NCBI. Furthermore, species with "Candidatus" and "sp." in their species names are filtered out. To disable filtering behavior, use the respective flag (see `xspect models train ncbi --help`).
 
 If you would like to train models with manually curated data from a directory, you can use:
 
@@ -82,6 +82,8 @@ To train models for MLST classifications, run:
 xspect models train mlst
 ```
 
+XspecT will prompt your for the organism name and the MLST scheme you would like to train a model for.
+
 ## Classification
 
 To classify samples, the command `xspect classify` can be used. This command will classify the sample based on the models available in your XspecT installation.
@@ -111,7 +113,7 @@ XspecT uses a kmer-based approach to classify samples. This means that the entir
 
 **Example**:
 ```bash
-xspect classify species --sparse-sampling-step 10 Acinetobacter path
+xspect classify species --sparse-sampling-step 10
 ```
 
 This will only consider every 10th kmer in the sample.
@@ -120,7 +122,7 @@ This will only consider every 10th kmer in the sample.
 By default, the classification results show only the taxonomy ID of each species along with its corresponding score for better readability. To display the full names associated with each taxonomy ID, you can use the `--display-names` (or `-n`) option:
 
 ```bash
-xspect classify species --display-names Acinetobacter path
+xspect classify species --display-names
 ```
 The output will then be formatted as: `Taxonomy_ID - Display_Name: Score` for each species.
 
@@ -132,6 +134,8 @@ Samples can also be classified based on Multi-locus sequence type schemas. To ML
 xspect classify mlst
 ```
 
+XspecT will prompt you for the organism, MLST scheme, and path to your sample directory.
+
 ## Filtering
 XspecT can also be used to filter samples based on their classification results. This is useful when analyzing metagenomic samples, for example when looking at genomic bycatch.
 

{xspect-0.6.0 → xspect-0.7.1}/docs/contributing.md

@@ -5,8 +5,8 @@ Thank you for your interest in contributing to XspecT! This page provides guidel
 
 When contributing to XspecT, please follow the following steps to ensure a smooth process:
 
-- **Read the documentation**: Familiarize yourself with the project by reading the [documentation](https://bionf.github.io/XspecT2/), including the [Understanding XspecT](understanding.md) page and the [architecture overview](#architecture-overview).
-- **Follow the coding standards**: Adhere to the project's coding standards and best practices. This includes using consistent naming conventions, writing clear and concise code, and documentation. Furthermore, please make sure your changes are algined with the project's [architecture](#architecture-overview).
+- **Read the documentation**: Familiarize yourself with the project by reading the [documentation](https://bionf.github.io/XspecT/), including the [Understanding XspecT](understanding.md) page and the [architecture overview](#architecture-overview).
+- **Follow the coding standards**: Adhere to the project's coding standards and best practices. This includes ensuring that your code is formatted using [Black](https://black.readthedocs.io/en/stable/) and linted with [Pylint](https://pylint.pycqa.org/en/latest/) for Python code, as well as using consistent naming conventions, writing clear and concise code, and documentation. Please use [pure functions](https://goodresearch.dev/decoupled#learn-to-identify-and-use-pure-functions) where possible and make sure your changes are aligned with the project's [architecture](#architecture-overview).
 - **Write tests**: Ensure that your changes are covered by tests. We use [pytest](https://docs.pytest.org/en/stable/) for testing. If you add new features or fix bugs, please include tests to verify your changes.
 - **Document your changes**: Update the documentation to reflect any new features or changes you make. This includes updating the README, Google-style docstrings, and the [Mkdocs](https://www.mkdocs.org)-based documentation.
 - **Use clear commit messages**: When committing your changes, use clear and descriptive commit messages that explain the purpose of the changes.
@@ -17,12 +17,12 @@ To set up XspecT for development, first make sure you have [Python](https://www.
 
 Get started by cloning the repository:
 ```bash
-git clone https://github.com/BIONF/XspecT2.git
+git clone https://github.com/BIONF/XspecT.git
 ```
 
 You then need to build the web application using Vite. Navigate to the `xspect-web` directory, install dependencies, and run the build command, which will also watch for changes:
 ```bash
-cd XspecT2/src/xspect/xspect-web
+cd XspecT/src/xspect/xspect-web
 ```
 ```bash
 npm i
@@ -86,7 +86,7 @@ We use GitHub Actions to run checks on commits and pull requests. These checks i
 
 Additionally, Github Actions are also used for deployment:
 
-- **Documentation**: The Mkdocs-based documentation is built and deployed to GitHub Pages on changes to the `main` branch. You can view the documentation at [https://bionf.github.io/XspecT2/](https://bionf.github.io/XspecT2/).
+- **Documentation**: The Mkdocs-based documentation is built and deployed to GitHub Pages on changes to the `main` branch. You can view the documentation at [https://bionf.github.io/XspecT/](https://bionf.github.io/XspecT/).
 - **Python package**: The Python package is built and uploaded to PyPI when a new release is created. This allows users to easily install the latest version of XspecT using `pip install xspect`. Pre-releases are uploaded to TestPyPI and can be installed using `pip install --index-url https://test.pypi.org/simple/ xspect`.
 
 ## Pull Request Process

{xspect-0.6.0 → xspect-0.7.1}/mkdocs.yml

@@ -7,7 +7,7 @@ theme:
 plugins:
   - include-markdown
   - search
-repo_url: https://github.com/BIONF/XspecT2
+repo_url: https://github.com/BIONF/XspecT
 markdown_extensions:
   - attr_list
 nav:

{xspect-0.6.0 → xspect-0.7.1}/pyproject.toml

@@ -1,10 +1,10 @@
 [project]
 name = "XspecT"
-version = "0.6.0"
+version = "0.7.1"
 description = "Tool to monitor and characterize pathogens using Bloom filters."
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.14"
 dependencies = [
   "biopython",
   "requests",

{xspect-0.6.0 → xspect-0.7.1}/scripts/benchmark/classify/main.nf

@@ -5,13 +5,14 @@ process classifySample {
 
   input:
   path sample
+  val model
 
   output:
   path "${sample.baseName}.json"
 
   script:
   """
-  xspect classify species -g Acinetobacter -i ${sample} -o ${sample.baseName}.json
+  xspect classify species -g ${model} -i ${sample} -o ${sample.baseName}.json
   """
 
   stub:

{xspect-0.6.0 → xspect-0.7.1}/scripts/benchmark/environment.yml

@@ -2,6 +2,7 @@ name: xspect-benchmark
 channels:
   - conda-forge
 dependencies:
+  - python=3.12
   - pip
   - pip:
-      - XspecT
+      - XspecT==0.5.4

{xspect-0.6.0 → xspect-0.7.1}/scripts/benchmark/main.nf

@@ -2,9 +2,56 @@
 
 include { classifySample as classifyAssembly } from './classify'
 include { classifySample as classifyRead } from './classify'
+include { strain_species_mapping } from '../nextflow-utils'
 
-process downloadModels {
-  conda "./scripts/benchmark/environment.yml"
+// --------------------- PARAMETERS ---------------------
+params.publishDir       = "results/benchmark"
+params.xspectModel     = "Acinetobacter"
+
+// --------------------- WORKFLOW -----------------------
+workflow {
+  species_model = getModelJSON()
+  name_mapping = getNameMapping(species_model)
+  genomes = file("data/genomes")
+  tax_report = file("data/aci_species.json")
+  tax_mapping_json = strain_species_mapping(tax_report)
+  assemblies = createAssemblyTable(genomes, tax_mapping_json, species_model)
+
+  // Whole genome assemblies
+  samples = Channel.fromPath("${genomes}/**/*.fna")
+    .flatten()
+  filtered_samples = assemblies
+    .splitCsv(header: true, sep: '\t')
+    .map { row -> row['Assembly Accession'] }
+    .cross(samples.map { sample -> 
+      [sample.baseName.split('_')[0..1].join('_'), sample]
+    })
+    .map { it[1][1] }
+  classifications = classifyAssembly(filtered_samples, params.xspectModel)
+  summarizeClassifications(assemblies, classifications.collect())
+  confusionMatrix(summarizeClassifications.out, name_mapping)
+  mismatchConfusionMatrix(summarizeClassifications.out, name_mapping)
+
+  // Simulated reads
+  selectForReadGen(assemblies, species_model)
+  read_assemblies = selectForReadGen.out
+    .splitCsv(header: true, sep: '\t')
+    .map { row -> row['Assembly Accession'] }
+    .cross(samples.map { sample -> 
+      [sample.baseName.split('_')[0..1].join('_'), sample]
+    })
+    .map { it[1][1] }
+  filterForChromosome(read_assemblies)
+  generateReads(filterForChromosome.out)
+  read_classifications = classifyRead(generateReads.out, params.xspectModel)
+  summarizeReadClassifications(selectForReadGen.out, read_classifications.collect())
+
+  calculateStats(summarizeClassifications.out, summarizeReadClassifications.out)
+  }
+
+// --------------------- PROCESSES ---------------------
+
+process getModelJSON {
   cpus 2
   memory '16 GB'
 
@@ -13,10 +60,8 @@ process downloadModels {
 
   script:
   """
-  if [ ! "$HOME/xspect-data/models/acinetobacter-species.json" ]; then
-    xspect models download
-  fi
-  cp "$HOME/xspect-data/models/acinetobacter-species.json" species_model.json
+  model_name="${params.xspectModel.toLowerCase().replaceAll('_','-')}-species.json"
+  cp "$HOME/xspect-data/models/\$model_name" species_model.json
   """
 }
 
@@ -50,7 +95,7 @@ process createAssemblyTable {
 
   input:
   path genomes
-  path tax_report
+  path tax_mapping_json
   path species_model
 
   output:
@@ -70,28 +115,11 @@ process createAssemblyTable {
   awk -F'\t' 'NR==1 || \$5 == "OK"' assemblies.tsv > assemblies_filtered.tsv
   mv assemblies_filtered.tsv assemblies.tsv
 
-  # map taxonmic IDs to species IDs (taxonomic IDs might be strain IDs)
-  jq '
-    .reports
-    | map(select(.taxonomy.children != null))
-    | map({
-        species_id: .taxonomy.tax_id,
-        children: .taxonomy.children
-      })
-    | map(
-        . as \$entry
-        | \$entry.children
-        | map({ (tostring): \$entry.species_id })
-        | add
-      )
-    | add
-  ' ${tax_report} > tax_mapping.json
-
   # add species IDs to assemblies.tsv
   declare -A species_map
   while IFS="=" read -r key val; do
     species_map["\$key"]="\$val"
-  done < <(jq -r 'to_entries[] | "\\(.key)=\\(.value)"' tax_mapping.json)
+  done < <(jq -r 'to_entries[] | "\\(.key)=\\(.value)"' ${tax_mapping_json})
 
   {
     IFS='\t' read -r -a header < assemblies.tsv
@@ -118,6 +146,21 @@ process createAssemblyTable {
   ' assemblies.tsv > temp_assemblies.tsv
   mv temp_assemblies.tsv assemblies.tsv
   rm valid_species.txt
+
+  # filter out assemblies that are part of the training set
+  jq -r '.training_accessions | to_entries[] | .value[]' ${species_model} > training_accessions.txt
+  awk -F'\t' '
+    BEGIN {
+      while ((getline acc < "training_accessions.txt") > 0) {
+        training[acc] = 1;
+      }
+      close("training_accessions.txt");
+    }
+    NR==1 { print; next }
+    !(\$1 in training) { print }
+  ' assemblies.tsv > temp_assemblies.tsv
+  mv temp_assemblies.tsv assemblies.tsv
+  rm training_accessions.txt
   """
 
   stub:
@@ -130,7 +173,7 @@ process summarizeClassifications {
   conda "conda-forge::pandas"
   cpus 4
   memory '16 GB'
-  publishDir "results"
+  publishDir { params.publishDir }, mode: 'copy'
 
   input:
   path assemblies
@@ -208,15 +251,32 @@ process selectForReadGen {
   ]
   assemblies = assemblies[~assemblies['Assembly Accession'].isin(training_accessions)]
 
-  # use up to three assemblies for each species
-  assemblies = assemblies.groupby('Species ID').head(3)
-
   assemblies.to_csv('selected_samples.tsv', sep='\\t', index=False)
   """
 }
 
+process filterForChromosome {
+  conda "bioconda::seqkit"
+  cpus 2
+  memory '16 GB'
+
+  input:
+  path sample
+
+  output:
+  path "${sample.baseName}_chromosome.fna"
+
+  script:
+  """
+  set -euo pipefail
+
+  seqkit sort -l -r ${sample} > sorted.tmp
+  seqkit head -n 1 sorted.tmp | seqkit seq -t dna -o "${sample.baseName}_chromosome.fna"
+  """
+}
+
 process generateReads {
-  conda "conda-forge::pandas conda-forge::biopython"
+  conda "bioconda::art"
   cpus 2
   memory '16 GB'
 
@@ -228,37 +288,24 @@ process generateReads {
 
   script:
   """
-  #!/usr/bin/env python
-  import random
-  from Bio import SeqIO
-  
-  read_length = 100
-  num_reads = 100000
-  seed = 42
-  
-  random.seed(seed)
-  sequences = list(SeqIO.parse("${sample}", "fasta"))
-  chromosome_sequence = max(sequences, key=len)  # we assume the longest sequence is the chromosome
-  
-  ch_rec_id = chromosome_sequence.id
-  ch_seq = chromosome_sequence.seq
-  ch_seqlen = len(chromosome_sequence.seq)
-  with open("${sample.baseName}_simulated.fq", "w") as f:
-    for i in range(num_reads):
-      start = random.randint(0, ch_seqlen - read_length)
-      read_seq = ch_seq[start:start + read_length]
-      f.write(f"@read_{i}_{ch_rec_id}_{start}-{start+read_length}\\n")
-      f.write(f"{read_seq}\\n")
-      f.write("+\\n")
-      f.write(f"{len(read_seq)*'~'}\\n")
+  set -euo pipefail
+
+  art_illumina \
+    -ss HS25 \
+    -i "${sample}" \
+    -l 125 \
+    -c 100000 \
+    -na \
+    -rs 42 \
+    -o "${sample.baseName}_simulated"
   """
 }
 
 process summarizeReadClassifications {
   conda "conda-forge::pandas"
   cpus 4
-  memory '16 GB'
-  publishDir "results"
+  memory '64 GB'
+  publishDir { params.publishDir }, mode: 'copy'
 
   input:
   path read_assemblies
@@ -278,10 +325,9 @@ process summarizeReadClassifications {
   
   # Create a mapping of accession to species ID
   accession_to_species = dict(zip(df_assemblies['Assembly Accession'], df_assemblies['Species ID']))
-
-  results = []
   
   classifications = '${read_classifications}'.split()
+  include_header = True
   for json_file in classifications:
     basename = os.path.basename(json_file).replace('.json', '')
     accession = '_'.join(basename.split('_')[:2])
@@ -291,6 +337,7 @@ process summarizeReadClassifications {
     with open(json_file, 'r') as f:
       data = json.load(f)
       scores = data.get('scores', {})
+      results = []
       
       for read_name, read_scores in scores.items():
         if read_name != 'total':
@@ -310,17 +357,20 @@ process summarizeReadClassifications {
               result[species] = score
 
             results.append(result)
+      
+
 
-  df_results = pd.DataFrame(results)
-  df_results.to_csv('read_classifications.tsv', sep='\\t', index=False)
+      df_results = pd.DataFrame(results)
+      df_results.to_csv('read_classifications.tsv', sep='\\t', index=False, mode='a', header=include_header)
+      include_header = False
   """
 }
 
 process calculateStats {
   conda "conda-forge::pandas conda-forge::scikit-learn"
-  cpus 2
-  memory '16 GB'
-  publishDir "results"
+  cpus 8
+  memory '256 GB'
+  publishDir { params.publishDir }, mode: 'copy'
 
   input:
   path assembly_classifications
@@ -399,7 +449,7 @@ process confusionMatrix {
   conda "conda-forge::pandas conda-forge::scikit-learn conda-forge::numpy conda-forge::matplotlib"
   cpus 2
   memory '16 GB'
-  publishDir "results"
+  publishDir { params.publishDir }, mode: 'copy'
 
   input:
   path classifications
@@ -449,7 +499,7 @@ process mismatchConfusionMatrix {
   conda "conda-forge::pandas conda-forge::scikit-learn conda-forge::numpy conda-forge::matplotlib"
   cpus 2
   memory '16 GB'
-  publishDir "results"
+  publishDir { params.publishDir }, mode: 'copy'
 
   input:
   path classifications
@@ -506,43 +556,4 @@ process mismatchConfusionMatrix {
   
   plt.savefig('mismatches_confusion_matrix.png', dpi=300, bbox_inches='tight')
   """
-}
-
-
-workflow {
-  species_model = downloadModels()
-  name_mapping = getNameMapping(species_model)
-  genomes = file("data/genomes")
-  tax_report = file("data/aci_species.json")
-  assemblies = createAssemblyTable(genomes, tax_report, species_model)
-
-  // Whole genome assemblies
-  samples = Channel.fromPath("${genomes}/**/*.fna")
-    .flatten()
-  filtered_samples = assemblies
-    .splitCsv(header: true, sep: '\t')
-    .map { row -> row['Assembly Accession'] }
-    .cross(samples.map { sample -> 
-      [sample.baseName.split('_')[0..1].join('_'), sample]
-    })
-    .map { it[1][1] }
-  classifications = classifyAssembly(filtered_samples)
-  summarizeClassifications(assemblies, classifications.collect())
-  confusionMatrix(summarizeClassifications.out, name_mapping)
-  mismatchConfusionMatrix(summarizeClassifications.out, name_mapping)
-
-  // Simulated reads
-  selectForReadGen(assemblies, species_model)
-  read_assemblies = selectForReadGen.out
-    .splitCsv(header: true, sep: '\t')
-    .map { row -> row['Assembly Accession'] }
-    .cross(samples.map { sample -> 
-      [sample.baseName.split('_')[0..1].join('_'), sample]
-    })
-    .map { it[1][1] }
-  generateReads(read_assemblies)
-  read_classifications = classifyRead(generateReads.out)
-  summarizeReadClassifications(selectForReadGen.out, read_classifications.collect())
-
-  calculateStats(summarizeClassifications.out, summarizeReadClassifications.out)
-  }
+}

{xspect-0.6.0 → xspect-0.7.1}/scripts/benchmark/nextflow.config

@@ -1,7 +1,6 @@
 process.executor = 'slurm'
 executor.account = 'intern'
 process.queue = 'all'
-executor.perCpuMemAllocation = true
 
 
 conda.enabled = true

{xspect-0.6.0 → xspect-0.7.1}/scripts/benchmark-data/download_data.slurm

@@ -5,7 +5,7 @@
 #SBATCH --mem-per-cpu=8gb             
 #SBATCH --job-name="download_acinetobacter"
 
-datasets download genome taxon 469 --filename acinetobacter_dataset.zip --assembly-source GenBank --assembly-version latest --exclude-atypical --dehydrated
+datasets download genome taxon 469 --filename acinetobacter_dataset.zip --assembly-source RefSeq --assembly-version latest --exclude-atypical --dehydrated
 unzip -o acinetobacter_dataset.zip -d genomes
 datasets rehydrate --directory genomes
 rm acinetobacter_dataset.zip

xspect-0.7.1/scripts/nextflow-utils/environment.yml

@@ -0,0 +1,8 @@
+name: xspect
+channels:
+  - conda-forge
+dependencies:
+  - python=3.12
+  - pip
+  - pip:
+      - XspecT==0.5.4