viralunity 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- viralunity/__init__.py +4 -0
- viralunity/_orchestrator.py +135 -0
- viralunity/_subprocess.py +52 -0
- viralunity/config_generator.py +587 -0
- viralunity/constants.py +151 -0
- viralunity/exceptions.py +100 -0
- viralunity/logging_config.py +101 -0
- viralunity/provenance.py +102 -0
- viralunity/scripts/__init__.py +0 -0
- viralunity/scripts/consensus_illumina.smk +142 -0
- viralunity/scripts/consensus_illumina_segmented.smk +203 -0
- viralunity/scripts/consensus_nanopore.smk +128 -0
- viralunity/scripts/consensus_nanopore_segmented.smk +172 -0
- viralunity/scripts/envs/alignment.yaml +11 -0
- viralunity/scripts/envs/assembly.yaml +6 -0
- viralunity/scripts/envs/clair3.yaml +7 -0
- viralunity/scripts/envs/consensus.yaml +10 -0
- viralunity/scripts/envs/genome_selection.yaml +8 -0
- viralunity/scripts/envs/medaka.yaml +6 -0
- viralunity/scripts/envs/qc.yaml +8 -0
- viralunity/scripts/envs/taxonomy.yaml +8 -0
- viralunity/scripts/envs/utils.yaml +10 -0
- viralunity/scripts/metagenomics_illumina.smk +279 -0
- viralunity/scripts/metagenomics_nanopore.smk +278 -0
- viralunity/scripts/python/__init__.py +0 -0
- viralunity/scripts/python/add_RPM_to_summary.py +143 -0
- viralunity/scripts/python/add_negative_control_enrichment.py +418 -0
- viralunity/scripts/python/add_rpkm_to_summary.py +125 -0
- viralunity/scripts/python/annotate_diamond_taxonomy.py +90 -0
- viralunity/scripts/python/apply_max_rpm_bleed_filter.py +172 -0
- viralunity/scripts/python/build_genome_length_table.py +197 -0
- viralunity/scripts/python/calculate_assembly_stats.py +151 -0
- viralunity/scripts/python/convert_diamond_output_to_krona_input.py +142 -0
- viralunity/scripts/python/filter_diamond_by_idxstats.py +131 -0
- viralunity/scripts/python/filter_krona_by_pass_taxids.py +224 -0
- viralunity/scripts/python/filter_taxids.py +82 -0
- viralunity/scripts/python/rename_sequences.py +67 -0
- viralunity/scripts/python/select_reference_genomes.py +470 -0
- viralunity/scripts/python/summarize_krona_taxa.py +163 -0
- viralunity/scripts/python/taxonomy.py +77 -0
- viralunity/scripts/rules/__init__.py +0 -0
- viralunity/scripts/rules/alignment_illumina.smk +73 -0
- viralunity/scripts/rules/alignment_nanopore.smk +66 -0
- viralunity/scripts/rules/consensus_illumina.smk +105 -0
- viralunity/scripts/rules/consensus_illumina_common.smk +86 -0
- viralunity/scripts/rules/consensus_nanopore.smk +67 -0
- viralunity/scripts/rules/consensus_nanopore_common.smk +57 -0
- viralunity/scripts/rules/metagenomics_assembly_illumina.smk +39 -0
- viralunity/scripts/rules/metagenomics_assembly_nanopore.smk +109 -0
- viralunity/scripts/rules/metagenomics_dehost_illumina.smk +130 -0
- viralunity/scripts/rules/metagenomics_dehost_nanopore.smk +97 -0
- viralunity/scripts/rules/metagenomics_diamond_contigs_illumina.smk +338 -0
- viralunity/scripts/rules/metagenomics_diamond_contigs_nanopore.smk +379 -0
- viralunity/scripts/rules/metagenomics_diamond_reads_illumina.smk +226 -0
- viralunity/scripts/rules/metagenomics_diamond_reads_nanopore.smk +226 -0
- viralunity/scripts/rules/metagenomics_genome_lengths.smk +37 -0
- viralunity/scripts/rules/metagenomics_kraken2_contigs_illumina.smk +210 -0
- viralunity/scripts/rules/metagenomics_kraken2_contigs_nanopore.smk +210 -0
- viralunity/scripts/rules/metagenomics_kraken2_reads_illumina.smk +214 -0
- viralunity/scripts/rules/metagenomics_kraken2_reads_nanopore.smk +213 -0
- viralunity/scripts/rules/metagenomics_multiqc_illumina.smk +16 -0
- viralunity/scripts/rules/metagenomics_reference_assembly.smk +151 -0
- viralunity/scripts/rules/qc_illumina.smk +65 -0
- viralunity/scripts/rules/stats.smk +28 -0
- viralunity/validators.py +614 -0
- viralunity/viralunity_build_deacon_index_cli.py +60 -0
- viralunity/viralunity_cli.py +60 -0
- viralunity/viralunity_consensus.py +160 -0
- viralunity/viralunity_consensus_cli.py +423 -0
- viralunity/viralunity_create_samplesheet.py +277 -0
- viralunity/viralunity_get_databases_cli.py +862 -0
- viralunity/viralunity_meta.py +200 -0
- viralunity/viralunity_meta_cli.py +469 -0
- viralunity/viralunity_setup_cli.py +212 -0
- viralunity-1.2.0.dist-info/METADATA +121 -0
- viralunity-1.2.0.dist-info/RECORD +80 -0
- viralunity-1.2.0.dist-info/WHEEL +5 -0
- viralunity-1.2.0.dist-info/entry_points.txt +2 -0
- viralunity-1.2.0.dist-info/licenses/LICENSE +21 -0
- viralunity-1.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,587 @@
|
|
|
1
|
+
"""Configuration file generation for ViralUnity pipelines."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from typing import Any, Dict, List, Optional, Union
|
|
5
|
+
|
|
6
|
+
import yaml
|
|
7
|
+
|
|
8
|
+
from viralunity.constants import ConfigKeys, DataType, ResourceDefaults
|
|
9
|
+
from viralunity.exceptions import ConfigurationError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ConfigGenerator:
|
|
13
|
+
"""Generates YAML configuration files for Snakemake workflows."""
|
|
14
|
+
|
|
15
|
+
# Section names used as comment headers in the output YAML
|
|
16
|
+
SECTION_PARAMETERS = "parameters"
|
|
17
|
+
SECTION_DATABASES = "databases"
|
|
18
|
+
SECTION_RESOURCES = "resources"
|
|
19
|
+
SECTION_REFERENCE_ASSEMBLY = "reference assembly"
|
|
20
|
+
|
|
21
|
+
def __init__(self, config_path: str):
|
|
22
|
+
"""Initialize config generator.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
config_path: Path where the config file will be written
|
|
26
|
+
"""
|
|
27
|
+
self.config_path = config_path
|
|
28
|
+
self.config: Dict[str, Any] = {}
|
|
29
|
+
# Track which section each key belongs to
|
|
30
|
+
self._sections: Dict[str, str] = {}
|
|
31
|
+
|
|
32
|
+
def _set(self, key: str, value: Any, section: str) -> None:
|
|
33
|
+
"""Set a config key and tag it to a section.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
key: Configuration key name
|
|
37
|
+
value: Configuration value
|
|
38
|
+
section: Section name (parameters, databases, resources)
|
|
39
|
+
"""
|
|
40
|
+
self.config[key] = value
|
|
41
|
+
self._sections[key] = section
|
|
42
|
+
|
|
43
|
+
def add_samples(self, samples: Dict[str, List[str]], data_type: str) -> None:
|
|
44
|
+
"""Add samples to configuration.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
samples: Dictionary mapping sample names to file paths
|
|
48
|
+
data_type: Type of sequencing data (illumina or nanopore)
|
|
49
|
+
"""
|
|
50
|
+
formatted_samples = {}
|
|
51
|
+
for sample_name, file_paths in samples.items():
|
|
52
|
+
key = f"sample-{sample_name}"
|
|
53
|
+
if data_type == DataType.ILLUMINA:
|
|
54
|
+
if len(file_paths) != 2:
|
|
55
|
+
raise ConfigurationError(
|
|
56
|
+
f"Illumina sample {sample_name} must have 2 files, "
|
|
57
|
+
f"found {len(file_paths)}"
|
|
58
|
+
)
|
|
59
|
+
# Store R1/R2 as a list rather than a space-joined string so a
|
|
60
|
+
# file path containing a space is not silently split by the
|
|
61
|
+
# workflow's ``.split()`` on the sample value.
|
|
62
|
+
formatted_samples[key] = [file_paths[0], file_paths[1]]
|
|
63
|
+
else:
|
|
64
|
+
if len(file_paths) != 1:
|
|
65
|
+
raise ConfigurationError(
|
|
66
|
+
f"Nanopore sample {sample_name} must have 1 file, "
|
|
67
|
+
f"found {len(file_paths)}"
|
|
68
|
+
)
|
|
69
|
+
formatted_samples[key] = [file_paths[0]]
|
|
70
|
+
|
|
71
|
+
self._set(ConfigKeys.SAMPLES, formatted_samples, self.SECTION_PARAMETERS)
|
|
72
|
+
self._set(ConfigKeys.DATA, data_type, self.SECTION_PARAMETERS)
|
|
73
|
+
|
|
74
|
+
def add_output(self, output_dir: str, run_name: str) -> None:
|
|
75
|
+
"""Add output directory to configuration.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
output_dir: Base output directory
|
|
79
|
+
run_name: Name of the run
|
|
80
|
+
"""
|
|
81
|
+
self._set(
|
|
82
|
+
ConfigKeys.OUTPUT,
|
|
83
|
+
os.path.join(output_dir, run_name, ""),
|
|
84
|
+
self.SECTION_PARAMETERS,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
def add_threads(self, threads: int) -> None:
|
|
88
|
+
"""Add thread count to configuration.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
threads: Number of threads
|
|
92
|
+
"""
|
|
93
|
+
self._set(ConfigKeys.THREADS, threads, self.SECTION_PARAMETERS)
|
|
94
|
+
|
|
95
|
+
def add_consensus_nanopore_settings(
|
|
96
|
+
self,
|
|
97
|
+
minimum_read_length: int,
|
|
98
|
+
af_threshold: float,
|
|
99
|
+
chunk_size: int,
|
|
100
|
+
clair3_model: str,
|
|
101
|
+
variant_quality: int,
|
|
102
|
+
variant_depth: int,
|
|
103
|
+
minimum_map_quality: int,
|
|
104
|
+
) -> None:
|
|
105
|
+
"""Add Nanopore consensus-specific settings to configuration.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
minimum_read_length: Minimum read length threshold
|
|
109
|
+
af_threshold: Allele frequency threshold to call a variant into consensus
|
|
110
|
+
chunk_size: Size of chunks to process [clair3]
|
|
111
|
+
clair3_model: Model to use for variant calling [clair3]
|
|
112
|
+
variant_quality: Minimum variant quality to call a variant into consensus [clair3]
|
|
113
|
+
variant_depth: Minimum alt allele depth to call a variant into consensus [clair3]
|
|
114
|
+
minimum_map_quality: Minimum map quality to call a variant into consensus [clair3]
|
|
115
|
+
"""
|
|
116
|
+
P = self.SECTION_PARAMETERS
|
|
117
|
+
self._set(ConfigKeys.MINIMUM_LENGTH, minimum_read_length, P)
|
|
118
|
+
self._set(ConfigKeys.AF_THRESHOLD, af_threshold, P)
|
|
119
|
+
self._set(ConfigKeys.CHUNK_SIZE, chunk_size, P)
|
|
120
|
+
self._set(ConfigKeys.CLAIR3_MODEL, clair3_model, P)
|
|
121
|
+
self._set(ConfigKeys.VARIANT_QUALITY, variant_quality, P)
|
|
122
|
+
self._set(ConfigKeys.VARIANT_DEPTH, variant_depth, P)
|
|
123
|
+
self._set(ConfigKeys.MINIMUM_MAP_QUALITY, minimum_map_quality, P)
|
|
124
|
+
|
|
125
|
+
def add_illumina_settings(
|
|
126
|
+
self,
|
|
127
|
+
adapters: str,
|
|
128
|
+
minimum_read_length: int,
|
|
129
|
+
trim_head: Optional[int] = None,
|
|
130
|
+
trim_tail: Optional[int] = None,
|
|
131
|
+
cut_front_mean_quality: int = 20,
|
|
132
|
+
cut_tail_mean_quality: int = 20,
|
|
133
|
+
cut_right_window_size: int = 4,
|
|
134
|
+
cut_right_mean_quality: int = 20,
|
|
135
|
+
af_threshold: float = 0.5,
|
|
136
|
+
af_isnv_threshold: float = 0.05,
|
|
137
|
+
run_isnv: bool = False,
|
|
138
|
+
) -> None:
|
|
139
|
+
"""Add Illumina-specific settings to configuration (fastp QC).
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
adapters: Path to adapters file or "NA" for auto-detection
|
|
143
|
+
minimum_read_length: Minimum read length threshold
|
|
144
|
+
trim_head: Bases to trim from 5'
|
|
145
|
+
trim_tail: Bases to trim from 3'
|
|
146
|
+
cut_front_mean_quality: fastp cut_front mean quality threshold
|
|
147
|
+
cut_tail_mean_quality: fastp cut_tail mean quality threshold
|
|
148
|
+
cut_right_window_size: fastp cut_right window size
|
|
149
|
+
cut_right_mean_quality: fastp cut_right mean quality threshold
|
|
150
|
+
af_threshold: Allele frequency threshold to call a variant into consensus
|
|
151
|
+
af_isnv_threshold: Minimum allele frequency threshold to call a variant into iSNV analysis
|
|
152
|
+
run_isnv: Whether to run iSNV analysis
|
|
153
|
+
"""
|
|
154
|
+
P = self.SECTION_PARAMETERS
|
|
155
|
+
self._set(ConfigKeys.ADAPTERS, adapters, P)
|
|
156
|
+
self._set(ConfigKeys.MINIMUM_LENGTH, minimum_read_length, P)
|
|
157
|
+
self._set(ConfigKeys.TRIM_HEAD, trim_head if trim_head is not None else 0, P)
|
|
158
|
+
self._set(ConfigKeys.TRIM_TAIL, trim_tail if trim_tail is not None else 0, P)
|
|
159
|
+
self._set(ConfigKeys.CUT_FRONT_MEAN_QUALITY, cut_front_mean_quality, P)
|
|
160
|
+
self._set(ConfigKeys.CUT_TAIL_MEAN_QUALITY, cut_tail_mean_quality, P)
|
|
161
|
+
self._set(ConfigKeys.CUT_RIGHT_WINDOW_SIZE, cut_right_window_size, P)
|
|
162
|
+
self._set(ConfigKeys.CUT_RIGHT_MEAN_QUALITY, cut_right_mean_quality, P)
|
|
163
|
+
self._set(ConfigKeys.AF_THRESHOLD, af_threshold, P)
|
|
164
|
+
self._set(ConfigKeys.AF_ISNV_THRESHOLD, af_isnv_threshold, P)
|
|
165
|
+
self._set(ConfigKeys.RUN_ISNV, run_isnv, P)
|
|
166
|
+
|
|
167
|
+
def add_consensus_settings(
|
|
168
|
+
self,
|
|
169
|
+
reference: Union[str, Dict[str, str]],
|
|
170
|
+
primer_scheme: str,
|
|
171
|
+
minimum_coverage: int,
|
|
172
|
+
minimap2_consensus_align_flags: str = "-a --sam-hit-only --secondary=no --score-N=0",
|
|
173
|
+
) -> None:
|
|
174
|
+
"""Add consensus-specific settings to configuration.
|
|
175
|
+
|
|
176
|
+
Args:
|
|
177
|
+
reference: Path to reference genome (str) or dict mapping
|
|
178
|
+
segment names to paths for segmented viruses
|
|
179
|
+
primer_scheme: Path to primer scheme file or "NA"
|
|
180
|
+
minimum_coverage: Minimum coverage for consensus
|
|
181
|
+
minimap2_consensus_align_flags: Flags passed to minimap2 when
|
|
182
|
+
re-aligning the per-sample consensus FASTA back to the
|
|
183
|
+
reference for the final multiple-sequence alignment. The
|
|
184
|
+
default keeps the historical behaviour.
|
|
185
|
+
"""
|
|
186
|
+
P = self.SECTION_PARAMETERS
|
|
187
|
+
self._set(ConfigKeys.REFERENCE, reference, P)
|
|
188
|
+
self._set(ConfigKeys.SCHEME, primer_scheme, P)
|
|
189
|
+
self._set(ConfigKeys.MINIMUM_DEPTH, minimum_coverage, P)
|
|
190
|
+
self._set(
|
|
191
|
+
ConfigKeys.MINIMAP2_CONSENSUS_ALIGN_FLAGS,
|
|
192
|
+
minimap2_consensus_align_flags,
|
|
193
|
+
P,
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
def add_metagenomics_settings(
|
|
197
|
+
self,
|
|
198
|
+
kraken2_database: str,
|
|
199
|
+
krona_database: str,
|
|
200
|
+
remove_human_reads: bool,
|
|
201
|
+
remove_unclassified_reads: bool,
|
|
202
|
+
host_reference: str = "NA",
|
|
203
|
+
deacon_index: str = "NA",
|
|
204
|
+
taxdump: str = "NA",
|
|
205
|
+
run_denovo_assembly: bool = False,
|
|
206
|
+
run_kraken2_reads: bool = True,
|
|
207
|
+
run_kraken2_contigs: bool = True,
|
|
208
|
+
run_diamond_reads: bool = False,
|
|
209
|
+
run_diamond_contigs: bool = False,
|
|
210
|
+
taxids: str = "NA",
|
|
211
|
+
diamond_database: str = "NA",
|
|
212
|
+
diamond_sensitivity: str = "sensitive",
|
|
213
|
+
evalue: float = 0.001,
|
|
214
|
+
bleed_fraction: float = 0.005,
|
|
215
|
+
negative_controls: Optional[List[str]] = None,
|
|
216
|
+
minimum_hit_group: int = 4,
|
|
217
|
+
diamond_max_target_seqs: int = 1,
|
|
218
|
+
kraken2_extra_flags: str = "--report-minimizer-data",
|
|
219
|
+
compute_rpkm: bool = False,
|
|
220
|
+
enrichment_pseudocount: float = 1.0,
|
|
221
|
+
z_score_threshold: float = 3.0,
|
|
222
|
+
log2_ratio_threshold: float = 1.0,
|
|
223
|
+
) -> None:
|
|
224
|
+
"""Add metagenomics-specific settings to configuration.
|
|
225
|
+
|
|
226
|
+
Args:
|
|
227
|
+
kraken2_database: Path to Kraken2 database
|
|
228
|
+
krona_database: Path to Krona database
|
|
229
|
+
remove_human_reads: Whether to remove human reads
|
|
230
|
+
remove_unclassified_reads: Whether to remove unclassified reads
|
|
231
|
+
host_reference: Path to host genome FASTA for dehosting (or "NA")
|
|
232
|
+
deacon_index: Path to Deacon minimizer index for host depletion (or "NA"). If set, used instead of host_reference for dehosting.
|
|
233
|
+
taxdump: Path to NCBI taxdump dir (nodes.dmp, names.dmp)
|
|
234
|
+
run_denovo_assembly: Whether to run MEGAHIT assembly
|
|
235
|
+
run_kraken2_reads: Whether to run Kraken2 on reads
|
|
236
|
+
run_kraken2_contigs: Whether to run Kraken2 on contigs (if assembly)
|
|
237
|
+
run_diamond_reads: Whether to run DIAMOND on reads
|
|
238
|
+
run_diamond_contigs: Whether to run DIAMOND on contigs (if assembly)
|
|
239
|
+
taxids: NCBI assembly summary (protein2taxid) for Diamond taxonomy (or "NA")
|
|
240
|
+
diamond_database: Path to Diamond DB (protein FASTA)
|
|
241
|
+
diamond_sensitivity: Diamond sensitivity (e.g. sensitive, mid-sensitive)
|
|
242
|
+
evalue: E-value threshold for Diamond
|
|
243
|
+
bleed_fraction: Max-RPM bleed filter fraction
|
|
244
|
+
negative_controls: Sample IDs to use as negative controls
|
|
245
|
+
minimum_hit_group: Kraken2 --minimum-hit-group (default: 4)
|
|
246
|
+
diamond_max_target_seqs: DIAMOND --max-target-seqs value (default 1).
|
|
247
|
+
kraken2_extra_flags: Extra flags appended to every kraken2
|
|
248
|
+
invocation alongside ``--threads`` and ``--minimum-hit-group``.
|
|
249
|
+
Defaults to ``--report-minimizer-data``; set to ``""`` to drop it.
|
|
250
|
+
compute_rpkm: Whether to compute RPKM (requires viral_genomes and
|
|
251
|
+
viral_taxids to be set). Derived automatically from
|
|
252
|
+
``viral_genomes != "NA"`` in the CLI layer.
|
|
253
|
+
enrichment_pseudocount: Pseudocount added to both numerator and
|
|
254
|
+
denominator when computing fold-enrichment and log2-ratio.
|
|
255
|
+
z_score_threshold: Minimum z-score to consider a hit above
|
|
256
|
+
background (used when ≥2 negative controls are present).
|
|
257
|
+
log2_ratio_threshold: Minimum log2-ratio to consider a hit above
|
|
258
|
+
background (used when exactly 1 negative control is present, or
|
|
259
|
+
when z-score is undefined due to zero control variance).
|
|
260
|
+
"""
|
|
261
|
+
P = self.SECTION_PARAMETERS
|
|
262
|
+
D = self.SECTION_DATABASES
|
|
263
|
+
# Database paths
|
|
264
|
+
self._set(ConfigKeys.KRAKEN2_DATABASE, kraken2_database, D)
|
|
265
|
+
self._set(ConfigKeys.KRONA_DATABASE, krona_database, D)
|
|
266
|
+
self._set(ConfigKeys.HOST_REFERENCE, host_reference, D)
|
|
267
|
+
self._set(ConfigKeys.DEACON_INDEX, deacon_index, D)
|
|
268
|
+
self._set(ConfigKeys.TAXDUMP, taxdump, D)
|
|
269
|
+
self._set(ConfigKeys.TAXIDS, taxids, D)
|
|
270
|
+
self._set(ConfigKeys.DIAMOND_DATABASE, diamond_database, D)
|
|
271
|
+
# Pipeline parameters
|
|
272
|
+
self._set(ConfigKeys.REMOVE_HUMAN_READS, remove_human_reads, P)
|
|
273
|
+
self._set(ConfigKeys.REMOVE_UNCLASSIFIED_READS, remove_unclassified_reads, P)
|
|
274
|
+
self._set(ConfigKeys.RUN_DENOVO_ASSEMBLY, run_denovo_assembly, P)
|
|
275
|
+
self._set(ConfigKeys.RUN_KRAKEN2_READS, run_kraken2_reads, P)
|
|
276
|
+
self._set(ConfigKeys.RUN_KRAKEN2_CONTIGS, run_kraken2_contigs, P)
|
|
277
|
+
self._set(ConfigKeys.RUN_DIAMOND_READS, run_diamond_reads, P)
|
|
278
|
+
self._set(ConfigKeys.RUN_DIAMOND_CONTIGS, run_diamond_contigs, P)
|
|
279
|
+
self._set(ConfigKeys.DIAMOND_SENSITIVITY, diamond_sensitivity, P)
|
|
280
|
+
self._set(ConfigKeys.EVALUE, evalue, P)
|
|
281
|
+
self._set(ConfigKeys.BLEED_FRACTION, bleed_fraction, P)
|
|
282
|
+
self._set(ConfigKeys.NEGATIVE_CONTROLS, negative_controls or [], P)
|
|
283
|
+
self._set(ConfigKeys.MINIMUM_HIT_GROUP, minimum_hit_group, P)
|
|
284
|
+
self._set(ConfigKeys.DIAMOND_MAX_TARGET_SEQS, diamond_max_target_seqs, P)
|
|
285
|
+
self._set(ConfigKeys.KRAKEN2_EXTRA_FLAGS, kraken2_extra_flags, P)
|
|
286
|
+
self._set(ConfigKeys.COMPUTE_RPKM, compute_rpkm, P)
|
|
287
|
+
self._set(ConfigKeys.ENRICHMENT_PSEUDOCOUNT, enrichment_pseudocount, P)
|
|
288
|
+
self._set(ConfigKeys.Z_SCORE_THRESHOLD, z_score_threshold, P)
|
|
289
|
+
self._set(ConfigKeys.LOG2_RATIO_THRESHOLD, log2_ratio_threshold, P)
|
|
290
|
+
|
|
291
|
+
def add_reference_assembly_settings(
|
|
292
|
+
self,
|
|
293
|
+
run_reference_assembly: bool = False,
|
|
294
|
+
method: Optional[str] = None,
|
|
295
|
+
source: Optional[str] = None,
|
|
296
|
+
reads_count: int = 100,
|
|
297
|
+
contigs_count: int = 1,
|
|
298
|
+
families: str = "Coronaviridae,Orthomyxoviridae,Flaviviridae,Herpesviridae,Papillomaviridae,Paramyxoviridae,Adenoviridae",
|
|
299
|
+
reference_selection_strategy: str = "taxid",
|
|
300
|
+
blast_qcov: int = 80,
|
|
301
|
+
blast_pident: int = 80,
|
|
302
|
+
viral_genomes: str = "databases/virus_genomes/viral.genomes.fasta",
|
|
303
|
+
viral_taxids: str = "databases/virus_genomes/genome2taxid.tsv",
|
|
304
|
+
) -> None:
|
|
305
|
+
"""Add settings for reference assembly on metagenomics hits."""
|
|
306
|
+
section = self.SECTION_REFERENCE_ASSEMBLY
|
|
307
|
+
self._set(ConfigKeys.RUN_REFERENCE_ASSEMBLY, run_reference_assembly, section)
|
|
308
|
+
self._set(ConfigKeys.REF_ASSEMBLY_METHOD, method, section)
|
|
309
|
+
self._set(ConfigKeys.REF_ASSEMBLY_SOURCE, source, section)
|
|
310
|
+
self._set(ConfigKeys.REF_ASSEMBLY_READS_COUNT, reads_count, section)
|
|
311
|
+
self._set(ConfigKeys.REF_ASSEMBLY_CONTIGS_COUNT, contigs_count, section)
|
|
312
|
+
self._set(
|
|
313
|
+
ConfigKeys.REF_ASSEMBLY_FAMILIES,
|
|
314
|
+
[f.strip() for f in families.split(",")],
|
|
315
|
+
section,
|
|
316
|
+
)
|
|
317
|
+
self._set(ConfigKeys.REF_SELECTION_STRATEGY, reference_selection_strategy, section)
|
|
318
|
+
self._set(ConfigKeys.REF_BLAST_QCOV, blast_qcov, section)
|
|
319
|
+
self._set(ConfigKeys.REF_BLAST_PIDENT, blast_pident, section)
|
|
320
|
+
|
|
321
|
+
db_section = self.SECTION_DATABASES
|
|
322
|
+
self._set(ConfigKeys.VIRAL_GENOMES, viral_genomes, db_section)
|
|
323
|
+
self._set(ConfigKeys.VIRAL_TAXIDS, viral_taxids, db_section)
|
|
324
|
+
|
|
325
|
+
def add_nanopore_settings(
|
|
326
|
+
self,
|
|
327
|
+
run_polish_racon: bool = False,
|
|
328
|
+
run_polish_medaka: bool = False,
|
|
329
|
+
medaka_model: Optional[str] = None,
|
|
330
|
+
clair3_model: Optional[str] = None,
|
|
331
|
+
) -> None:
|
|
332
|
+
"""Add Nanopore-specific settings (polishing: Racon, Medaka; clair3 model).
|
|
333
|
+
|
|
334
|
+
Args:
|
|
335
|
+
run_polish_racon: Whether to run Racon polishing after MEGAHIT.
|
|
336
|
+
run_polish_medaka: Whether to run Medaka polishing (after Racon if both enabled).
|
|
337
|
+
medaka_model: Medaka model name (e.g. r941_min_high_g360). Optional; Medaka uses default if not set.
|
|
338
|
+
clair3_model: Clair3 model name used by the reference-assembly
|
|
339
|
+
consensus path. Optional; the rule falls back to its own
|
|
340
|
+
default if not set.
|
|
341
|
+
"""
|
|
342
|
+
P = self.SECTION_PARAMETERS
|
|
343
|
+
self._set(ConfigKeys.RUN_POLISH_RACON, run_polish_racon, P)
|
|
344
|
+
self._set(ConfigKeys.RUN_POLISH_MEDAKA, run_polish_medaka, P)
|
|
345
|
+
if medaka_model is not None:
|
|
346
|
+
self._set(ConfigKeys.MEDAKA_MODEL, medaka_model, P)
|
|
347
|
+
if clair3_model is not None:
|
|
348
|
+
self._set(ConfigKeys.CLAIR3_MODEL, clair3_model, P)
|
|
349
|
+
|
|
350
|
+
def add_workflow_path(self, workflow_path: str) -> None:
|
|
351
|
+
"""Add workflow path to configuration.
|
|
352
|
+
|
|
353
|
+
Args:
|
|
354
|
+
workflow_path: Path to the workflow directory
|
|
355
|
+
"""
|
|
356
|
+
self._set("workflow_path", workflow_path, self.SECTION_PARAMETERS)
|
|
357
|
+
|
|
358
|
+
def add_resource_settings(self, args: Dict[str, Any], rule_names: list) -> None:
|
|
359
|
+
"""Add per-rule resource settings (CPUs and RAM) to configuration.
|
|
360
|
+
|
|
361
|
+
For each rule name in rule_names, writes ``<rule>_cpus`` and
|
|
362
|
+
``<rule>_ram`` keys to the config dict. Values are taken from
|
|
363
|
+
*args* if present; otherwise the defaults from
|
|
364
|
+
``ResourceDefaults`` are used.
|
|
365
|
+
|
|
366
|
+
Args:
|
|
367
|
+
args: Dictionary of pipeline arguments (from the CLI).
|
|
368
|
+
rule_names: List of Snakemake rule name strings that should
|
|
369
|
+
receive resource entries.
|
|
370
|
+
"""
|
|
371
|
+
R = self.SECTION_RESOURCES
|
|
372
|
+
for rule in rule_names:
|
|
373
|
+
cpus_key = f"{rule}_cpus"
|
|
374
|
+
ram_key = f"{rule}_ram"
|
|
375
|
+
self._set(cpus_key, args.get(cpus_key, ResourceDefaults.DEFAULT_CPUS), R)
|
|
376
|
+
self._set(ram_key, args.get(ram_key, ResourceDefaults.DEFAULT_RAM), R)
|
|
377
|
+
|
|
378
|
+
@classmethod
|
|
379
|
+
def write_skeleton(
|
|
380
|
+
cls,
|
|
381
|
+
pipeline: str,
|
|
382
|
+
data_type: str,
|
|
383
|
+
config_path: str,
|
|
384
|
+
placeholder_dir: str,
|
|
385
|
+
) -> str:
|
|
386
|
+
"""Write a placeholder YAML config that lets Snakemake parse a workflow.
|
|
387
|
+
|
|
388
|
+
Used by ``viralunity setup`` to materialize per-rule conda envs via
|
|
389
|
+
``snakemake(..., conda_create_envs_only=True)``. Even in
|
|
390
|
+
envs-only mode Snakemake walks the DAG and verifies that rule
|
|
391
|
+
inputs exist, so the caller is expected to populate
|
|
392
|
+
``placeholder_dir`` with empty files for the FASTQs, references,
|
|
393
|
+
and database paths emitted here. ``viralunity_setup_cli`` does
|
|
394
|
+
this for the package; tests use the same convention.
|
|
395
|
+
|
|
396
|
+
Args:
|
|
397
|
+
pipeline: ``"consensus"`` or ``"metagenomics"``.
|
|
398
|
+
data_type: ``"illumina"`` or ``"nanopore"``.
|
|
399
|
+
config_path: Where to write the YAML.
|
|
400
|
+
placeholder_dir: Directory that contains the placeholder input
|
|
401
|
+
files referenced by the generated config.
|
|
402
|
+
|
|
403
|
+
Returns:
|
|
404
|
+
``config_path``.
|
|
405
|
+
"""
|
|
406
|
+
gen = cls(config_path)
|
|
407
|
+
root = placeholder_dir.rstrip("/")
|
|
408
|
+
|
|
409
|
+
if data_type == DataType.ILLUMINA:
|
|
410
|
+
placeholder_samples = {
|
|
411
|
+
"skel": [
|
|
412
|
+
f"{root}/reads/skel_R1.fastq.gz",
|
|
413
|
+
f"{root}/reads/skel_R2.fastq.gz",
|
|
414
|
+
],
|
|
415
|
+
}
|
|
416
|
+
else:
|
|
417
|
+
placeholder_samples = {"skel": [f"{root}/reads/skel.fastq.gz"]}
|
|
418
|
+
|
|
419
|
+
gen.add_samples(placeholder_samples, data_type)
|
|
420
|
+
gen.add_output(f"{root}/output", "skeleton_run")
|
|
421
|
+
gen.add_threads(1)
|
|
422
|
+
|
|
423
|
+
# Every optional feature flag is enabled below so Snakemake's DAG
|
|
424
|
+
# walk reaches every rule and ``conda_create_envs_only=True``
|
|
425
|
+
# materializes every per-rule env that pipeline could possibly need.
|
|
426
|
+
# If a flag is left off here, its env is silently skipped by
|
|
427
|
+
# ``viralunity setup`` and falls back to dynamic env creation at
|
|
428
|
+
# the user's first real run — exactly the failure mode setup
|
|
429
|
+
# exists to prevent.
|
|
430
|
+
if pipeline == "consensus":
|
|
431
|
+
gen.add_consensus_settings(
|
|
432
|
+
reference=f"{root}/references/skel.reference.fasta",
|
|
433
|
+
primer_scheme="NA",
|
|
434
|
+
minimum_coverage=20,
|
|
435
|
+
)
|
|
436
|
+
if data_type == DataType.ILLUMINA:
|
|
437
|
+
# run_isnv=True pulls the detect_isnv (LoFreq) rule into
|
|
438
|
+
# the DAG, which uses envs/consensus.yaml.
|
|
439
|
+
gen.add_illumina_settings(
|
|
440
|
+
adapters="NA",
|
|
441
|
+
minimum_read_length=50,
|
|
442
|
+
run_isnv=True,
|
|
443
|
+
)
|
|
444
|
+
else:
|
|
445
|
+
gen.add_consensus_nanopore_settings(
|
|
446
|
+
minimum_read_length=50,
|
|
447
|
+
af_threshold=0.51,
|
|
448
|
+
chunk_size=10000,
|
|
449
|
+
clair3_model="r1041_e82_400bps_sup_v500",
|
|
450
|
+
variant_quality=20,
|
|
451
|
+
variant_depth=10,
|
|
452
|
+
minimum_map_quality=30,
|
|
453
|
+
)
|
|
454
|
+
gen.add_workflow_path(".")
|
|
455
|
+
elif pipeline == "metagenomics":
|
|
456
|
+
if data_type == DataType.ILLUMINA:
|
|
457
|
+
gen.add_illumina_settings(adapters="NA", minimum_read_length=50)
|
|
458
|
+
gen.add_metagenomics_settings(
|
|
459
|
+
kraken2_database=f"{root}/kraken2/",
|
|
460
|
+
krona_database=f"{root}/krona/taxonomy/",
|
|
461
|
+
remove_human_reads=False,
|
|
462
|
+
remove_unclassified_reads=False,
|
|
463
|
+
taxdump=f"{root}/taxdump/",
|
|
464
|
+
taxids=f"{root}/diamond/protein2taxid.tsv",
|
|
465
|
+
diamond_database=f"{root}/diamond/nr.dmnd",
|
|
466
|
+
# All four classifier toggles on => kraken2 + diamond
|
|
467
|
+
# rules on reads *and* contigs are in the DAG; pulls
|
|
468
|
+
# taxonomy.yaml.
|
|
469
|
+
run_denovo_assembly=True,
|
|
470
|
+
run_kraken2_reads=True,
|
|
471
|
+
run_kraken2_contigs=True,
|
|
472
|
+
run_diamond_reads=True,
|
|
473
|
+
run_diamond_contigs=True,
|
|
474
|
+
)
|
|
475
|
+
# Reference-assembly checkpoint pulls genome_selection.yaml.
|
|
476
|
+
gen.add_reference_assembly_settings(
|
|
477
|
+
run_reference_assembly=True,
|
|
478
|
+
method="kraken2",
|
|
479
|
+
source="reads",
|
|
480
|
+
viral_genomes=f"{root}/virus_genomes/viral.genomes.fasta",
|
|
481
|
+
viral_taxids=f"{root}/virus_genomes/genome2taxid.tsv",
|
|
482
|
+
)
|
|
483
|
+
if data_type == DataType.NANOPORE:
|
|
484
|
+
# Polish flags pull racon (consensus.yaml) and medaka.yaml.
|
|
485
|
+
gen.add_nanopore_settings(
|
|
486
|
+
run_polish_racon=True,
|
|
487
|
+
run_polish_medaka=True,
|
|
488
|
+
)
|
|
489
|
+
else:
|
|
490
|
+
raise ValueError(
|
|
491
|
+
f"Unknown pipeline: {pipeline!r} (expected 'consensus' or 'metagenomics')"
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
gen.save()
|
|
495
|
+
return config_path
|
|
496
|
+
|
|
497
|
+
# Empty files (relative to ``placeholder_dir``) that
|
|
498
|
+
# ``ConfigGenerator.write_skeleton`` expects to exist on disk for
|
|
499
|
+
# ``snakemake --conda-create-envs-only`` to succeed. Kept here so
|
|
500
|
+
# the skeleton and its placeholder-file expectations stay in lock-step.
|
|
501
|
+
SKELETON_PLACEHOLDERS: Dict[str, Dict[str, List[str]]] = {
|
|
502
|
+
"consensus": {
|
|
503
|
+
"illumina": [
|
|
504
|
+
"reads/skel_R1.fastq.gz",
|
|
505
|
+
"reads/skel_R2.fastq.gz",
|
|
506
|
+
"references/skel.reference.fasta",
|
|
507
|
+
],
|
|
508
|
+
"nanopore": [
|
|
509
|
+
"reads/skel.fastq.gz",
|
|
510
|
+
"references/skel.reference.fasta",
|
|
511
|
+
],
|
|
512
|
+
},
|
|
513
|
+
"metagenomics": {
|
|
514
|
+
"illumina": [
|
|
515
|
+
"reads/skel_R1.fastq.gz",
|
|
516
|
+
"reads/skel_R2.fastq.gz",
|
|
517
|
+
"kraken2/hash.k2d",
|
|
518
|
+
"krona/taxonomy/taxonomy.tab",
|
|
519
|
+
"taxdump/nodes.dmp",
|
|
520
|
+
"taxdump/names.dmp",
|
|
521
|
+
"diamond/nr.dmnd",
|
|
522
|
+
"diamond/protein2taxid.tsv",
|
|
523
|
+
"virus_genomes/viral.genomes.fasta",
|
|
524
|
+
"virus_genomes/genome2taxid.tsv",
|
|
525
|
+
],
|
|
526
|
+
"nanopore": [
|
|
527
|
+
"reads/skel.fastq.gz",
|
|
528
|
+
"kraken2/hash.k2d",
|
|
529
|
+
"krona/taxonomy/taxonomy.tab",
|
|
530
|
+
"taxdump/nodes.dmp",
|
|
531
|
+
"taxdump/names.dmp",
|
|
532
|
+
"diamond/nr.dmnd",
|
|
533
|
+
"diamond/protein2taxid.tsv",
|
|
534
|
+
"virus_genomes/viral.genomes.fasta",
|
|
535
|
+
"virus_genomes/genome2taxid.tsv",
|
|
536
|
+
],
|
|
537
|
+
},
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
def save(self) -> None:
|
|
541
|
+
"""Save configuration to YAML file with section comment headers.
|
|
542
|
+
|
|
543
|
+
Keys are grouped into sections (# parameters, # databases,
|
|
544
|
+
# resources) based on the tags assigned by ``_set()``. Within
|
|
545
|
+
each section the insertion order of keys is preserved.
|
|
546
|
+
|
|
547
|
+
Raises:
|
|
548
|
+
ConfigurationError: If config directory cannot be created
|
|
549
|
+
"""
|
|
550
|
+
config_dir = os.path.dirname(self.config_path)
|
|
551
|
+
if config_dir: # Only create directory if path contains a directory component
|
|
552
|
+
os.makedirs(config_dir, exist_ok=True)
|
|
553
|
+
|
|
554
|
+
# Group keys by section, preserving insertion order
|
|
555
|
+
section_order = [
|
|
556
|
+
self.SECTION_PARAMETERS,
|
|
557
|
+
self.SECTION_REFERENCE_ASSEMBLY,
|
|
558
|
+
self.SECTION_DATABASES,
|
|
559
|
+
self.SECTION_RESOURCES,
|
|
560
|
+
]
|
|
561
|
+
grouped: Dict[str, Dict[str, Any]] = {s: {} for s in section_order}
|
|
562
|
+
|
|
563
|
+
for key, value in self.config.items():
|
|
564
|
+
section = self._sections.get(key, self.SECTION_PARAMETERS)
|
|
565
|
+
grouped[section][key] = value
|
|
566
|
+
|
|
567
|
+
try:
|
|
568
|
+
with open(self.config_path, "w") as f:
|
|
569
|
+
first = True
|
|
570
|
+
for section in section_order:
|
|
571
|
+
items = grouped[section]
|
|
572
|
+
if not items:
|
|
573
|
+
continue
|
|
574
|
+
if not first:
|
|
575
|
+
f.write("\n")
|
|
576
|
+
f.write(f"# {section}\n")
|
|
577
|
+
yaml.dump(
|
|
578
|
+
items,
|
|
579
|
+
f,
|
|
580
|
+
default_flow_style=False,
|
|
581
|
+
sort_keys=False,
|
|
582
|
+
)
|
|
583
|
+
first = False
|
|
584
|
+
except (OSError, IOError) as e:
|
|
585
|
+
raise ConfigurationError(
|
|
586
|
+
f"Failed to write config file to {self.config_path}: {e}"
|
|
587
|
+
) from e
|