PythiaLabelGenerator 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- labelgenerator/__init__.py +15 -0
- labelgenerator/iqtree.py +142 -0
- labelgenerator/iqtree_parser.py +183 -0
- labelgenerator/label.py +231 -0
- labelgenerator/logger.py +32 -0
- labelgenerator/main.py +176 -0
- labelgenerator/raxmlng.py +173 -0
- pythialabelgenerator-1.0.0.dist-info/METADATA +215 -0
- pythialabelgenerator-1.0.0.dist-info/RECORD +12 -0
- pythialabelgenerator-1.0.0.dist-info/WHEEL +4 -0
- pythialabelgenerator-1.0.0.dist-info/entry_points.txt +2 -0
- pythialabelgenerator-1.0.0.dist-info/licenses/LICENSE +674 -0
labelgenerator/main.py
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import pathlib
|
|
3
|
+
import shutil
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from labelgenerator import __version__
|
|
8
|
+
from labelgenerator.label import compute_label
|
|
9
|
+
from labelgenerator.logger import (
|
|
10
|
+
SCRIPT_START,
|
|
11
|
+
get_header,
|
|
12
|
+
log_runtime_information,
|
|
13
|
+
logger,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
DEFAULT_RAXMLNG_EXE = (
|
|
17
|
+
pathlib.Path(shutil.which("raxml-ng")) if shutil.which("raxml-ng") else None
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
DEFAULT_IQTREE_EXE = (
|
|
21
|
+
pathlib.Path(shutil.which("iqtree2")) if shutil.which("iqtree2") else None
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _parse_cli():
|
|
26
|
+
parser = argparse.ArgumentParser(
|
|
27
|
+
description="Generate the ground truth difficulty for the given MSA."
|
|
28
|
+
)
|
|
29
|
+
parser.add_argument(
|
|
30
|
+
"-m",
|
|
31
|
+
"--msa",
|
|
32
|
+
type=str,
|
|
33
|
+
required=True,
|
|
34
|
+
help="Multiple Sequence Alignment to compute the ground truth difficulty for. "
|
|
35
|
+
"Must be in either phylip or fasta format.",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
parser.add_argument(
|
|
39
|
+
"-r",
|
|
40
|
+
"--raxmlng",
|
|
41
|
+
type=str,
|
|
42
|
+
default=DEFAULT_RAXMLNG_EXE,
|
|
43
|
+
required=DEFAULT_RAXMLNG_EXE is None,
|
|
44
|
+
help="Path to the binary of RAxML-NG. For install instructions see https://github.com/amkozlov/raxml-ng."
|
|
45
|
+
"(default: 'raxml-ng' if in $PATH, otherwise this option is mandatory).",
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
parser.add_argument(
|
|
49
|
+
"-i",
|
|
50
|
+
"--iqtree",
|
|
51
|
+
type=str,
|
|
52
|
+
default=DEFAULT_IQTREE_EXE,
|
|
53
|
+
required=DEFAULT_IQTREE_EXE is None,
|
|
54
|
+
help="Path to the binary of IQ-TREE2. For install instructions see http://www.iqtree.org."
|
|
55
|
+
"(default: 'iqtree2' if in $PATH, otherwise this option is mandatory).",
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
parser.add_argument(
|
|
59
|
+
"-t",
|
|
60
|
+
"--threads",
|
|
61
|
+
type=int,
|
|
62
|
+
required=False,
|
|
63
|
+
help="Number of threads to use for the RAxML-NG tree inference and IQ-TREE statistical tests (default: autoconfig in RAxML-NG and IQ-TREE).",
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
parser.add_argument(
|
|
67
|
+
"-s",
|
|
68
|
+
"--seed",
|
|
69
|
+
type=int,
|
|
70
|
+
default=0,
|
|
71
|
+
required=False,
|
|
72
|
+
help="Seed for the RAxML-NG tree inference (default: 0).",
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
parser.add_argument(
|
|
76
|
+
"-p",
|
|
77
|
+
"--prefix",
|
|
78
|
+
type=str,
|
|
79
|
+
required=False,
|
|
80
|
+
help="Prefix of the RAxML-NG and IQ-TREE log and result files (default: MSA file name).",
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
parser.add_argument(
|
|
84
|
+
"--model",
|
|
85
|
+
type=str,
|
|
86
|
+
required=False,
|
|
87
|
+
help="Model to use for the RAxML-NG tree inference (default: 'GTR+G' for DNA, 'LG+G' for AA, "
|
|
88
|
+
"and 'MULTIx_GTR' for morphological data).",
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
parser.add_argument(
|
|
92
|
+
"--ntrees",
|
|
93
|
+
type=int,
|
|
94
|
+
required=False,
|
|
95
|
+
default=100,
|
|
96
|
+
help="Number of ML trees to infer (default: 100)",
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
parser.add_argument(
|
|
100
|
+
"--redo",
|
|
101
|
+
action="store_true",
|
|
102
|
+
help="Redo all computations, even if the results already exist.",
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
parser.add_argument(
|
|
106
|
+
"-V",
|
|
107
|
+
"--version",
|
|
108
|
+
action="version",
|
|
109
|
+
version=__version__,
|
|
110
|
+
help="Print the version number and exit.",
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
return parser.parse_args()
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def main():
|
|
117
|
+
logger.info(get_header())
|
|
118
|
+
args = _parse_cli()
|
|
119
|
+
|
|
120
|
+
msa_file = pathlib.Path(args.msa)
|
|
121
|
+
prefix = pathlib.Path(args.prefix) if args.prefix else msa_file
|
|
122
|
+
|
|
123
|
+
log_file = pathlib.Path(f"{prefix}.labelGen.log")
|
|
124
|
+
logger.add(log_file, format="{message}")
|
|
125
|
+
log_file.write_text(get_header() + "\n")
|
|
126
|
+
|
|
127
|
+
logger.info(
|
|
128
|
+
f"LabelGenerator was called at {time.strftime('%d-%b-%Y %H:%M:%S')} as follows:\n"
|
|
129
|
+
)
|
|
130
|
+
logger.info(" ".join(sys.argv))
|
|
131
|
+
logger.info("")
|
|
132
|
+
|
|
133
|
+
log_runtime_information("Starting label computation.")
|
|
134
|
+
|
|
135
|
+
difficulty = compute_label(
|
|
136
|
+
msa_file=msa_file,
|
|
137
|
+
raxmlng=pathlib.Path(args.raxmlng),
|
|
138
|
+
iqtree=pathlib.Path(args.iqtree),
|
|
139
|
+
prefix=prefix,
|
|
140
|
+
model=args.model,
|
|
141
|
+
n_trees=args.ntrees,
|
|
142
|
+
seed=args.seed,
|
|
143
|
+
threads=args.threads,
|
|
144
|
+
redo=args.redo,
|
|
145
|
+
log_info=True,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
script_end = time.perf_counter()
|
|
149
|
+
|
|
150
|
+
logger.info("")
|
|
151
|
+
logger.info(f"Ground Truth Difficulty for {msa_file}: {difficulty:.3f}")
|
|
152
|
+
|
|
153
|
+
if args.ntrees < 100:
|
|
154
|
+
logger.info(
|
|
155
|
+
"WARNING: The number of inferred ML trees is less than 100. The computed label may be less reliable."
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
logger.info("")
|
|
159
|
+
total_runtime = script_end - SCRIPT_START
|
|
160
|
+
hours, remainder = divmod(total_runtime, 3600)
|
|
161
|
+
minutes, seconds = divmod(remainder, 60)
|
|
162
|
+
|
|
163
|
+
if hours > 0:
|
|
164
|
+
logger.info(
|
|
165
|
+
f"Total runtime: {int(hours):02d}:{int(minutes):02d}:{seconds:02d} hours ({round(total_runtime)} seconds)."
|
|
166
|
+
)
|
|
167
|
+
elif minutes > 0:
|
|
168
|
+
logger.info(
|
|
169
|
+
f"Total runtime: {int(minutes):02d}:{int(seconds):02d} minutes ({round(total_runtime)} seconds)."
|
|
170
|
+
)
|
|
171
|
+
else:
|
|
172
|
+
logger.info(f"Total runtime: {seconds:.2f} seconds.")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
if __name__ == "__main__":
|
|
176
|
+
main()
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import pathlib
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from pypythia.raxmlng import RAxMLNG, get_raxmlng_rfdist_results, run_raxmlng_command
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _inference_results_exist_and_correct(prefix: pathlib.Path, n_trees: int) -> bool:
|
|
9
|
+
ml_trees = pathlib.Path(f"{prefix}.raxml.mlTrees")
|
|
10
|
+
best_tree = pathlib.Path(f"{prefix}.raxml.bestTree")
|
|
11
|
+
|
|
12
|
+
if n_trees == 1:
|
|
13
|
+
ml_trees = best_tree
|
|
14
|
+
|
|
15
|
+
logfile = pathlib.Path(f"{prefix}.raxml.log")
|
|
16
|
+
|
|
17
|
+
files_exist = ml_trees.exists() and best_tree.exists() and logfile.exists()
|
|
18
|
+
if not files_exist:
|
|
19
|
+
# Files don't exist yet, nothing to check.
|
|
20
|
+
return False
|
|
21
|
+
|
|
22
|
+
# Check if the number of ML trees is correct, if there is a mismatch, raise an error
|
|
23
|
+
n_trees_in_file = sum(1 for _ in ml_trees.open())
|
|
24
|
+
if n_trees_in_file != n_trees:
|
|
25
|
+
raise ValueError(
|
|
26
|
+
f"Number of trees in {ml_trees} ({n_trees_in_file}) does not match the expected number of trees ({n_trees})."
|
|
27
|
+
f"Please set the `redo` flag to recompute the trees."
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
# Finally, check if the previous RAxML-NG run completed successfully
|
|
31
|
+
return "Elapsed time:" in logfile.read_text()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def infer_ml_trees(
|
|
35
|
+
msa: pathlib.Path,
|
|
36
|
+
raxmlng: pathlib.Path,
|
|
37
|
+
model: str,
|
|
38
|
+
prefix: pathlib.Path,
|
|
39
|
+
n_trees: int = 100,
|
|
40
|
+
seed: int = 0,
|
|
41
|
+
threads: Optional[int] = None,
|
|
42
|
+
redo: bool = False,
|
|
43
|
+
) -> None:
|
|
44
|
+
"""
|
|
45
|
+
Infers ML trees using RAxML-NG.
|
|
46
|
+
|
|
47
|
+
If the results for the given prefix and number of trees already exist, the function will return without doing anything.
|
|
48
|
+
If the number of trees in the existing file does not match the expected number of trees, a ValueError is raised.
|
|
49
|
+
If you want to redo the computation, set redo=True.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
msa (pathlib.Path): Path to the MSA file.
|
|
53
|
+
raxmlng (pathlib.Path): Path to the RAxML-NG executable.
|
|
54
|
+
model (str): Model to use for the RAxML-NG tree inference.
|
|
55
|
+
prefix (pathlib.Path): Prefix to use for the RAxML-NG output files.
|
|
56
|
+
n_trees (int): Number of ML trees to infer.
|
|
57
|
+
seed (int): Seed to use for the RAxML-NG inference.
|
|
58
|
+
threads (Optional[int]): Number of threads to use for the RAxML-NG inference.
|
|
59
|
+
Per default, uses the automatic setting of RAxML-NG which is likely to use all cores of your machine.
|
|
60
|
+
redo (bool): Flag to redo the computation even if the results already exist.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
None
|
|
64
|
+
|
|
65
|
+
Raises:
|
|
66
|
+
ValueError:
|
|
67
|
+
- If the number of trees is less than 1.
|
|
68
|
+
- If the number of trees in a previously computed file does not match the expected number of trees.
|
|
69
|
+
|
|
70
|
+
"""
|
|
71
|
+
if n_trees < 1:
|
|
72
|
+
raise ValueError("Number of trees needs to be at least 1.")
|
|
73
|
+
|
|
74
|
+
if not redo and _inference_results_exist_and_correct(prefix, n_trees):
|
|
75
|
+
return
|
|
76
|
+
|
|
77
|
+
n_pars_trees = math.ceil(n_trees / 2)
|
|
78
|
+
n_rand_trees = n_trees - n_pars_trees
|
|
79
|
+
rand_string = f",rand{{{n_rand_trees}}}" if n_rand_trees > 0 else ""
|
|
80
|
+
|
|
81
|
+
cmd = [
|
|
82
|
+
raxmlng,
|
|
83
|
+
"--msa",
|
|
84
|
+
msa,
|
|
85
|
+
"--model",
|
|
86
|
+
model,
|
|
87
|
+
"--seed",
|
|
88
|
+
seed,
|
|
89
|
+
"--prefix",
|
|
90
|
+
prefix,
|
|
91
|
+
"--tree",
|
|
92
|
+
f"pars{{{n_pars_trees}}}{rand_string}",
|
|
93
|
+
]
|
|
94
|
+
|
|
95
|
+
if threads is not None:
|
|
96
|
+
cmd.extend(["--threads", threads])
|
|
97
|
+
|
|
98
|
+
if redo:
|
|
99
|
+
cmd.append("--redo")
|
|
100
|
+
|
|
101
|
+
run_raxmlng_command(list(map(str, cmd)))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _rfdist_results_exists_and_correct(prefix: pathlib.Path, n_trees: int) -> bool:
|
|
105
|
+
rfdist = pathlib.Path(f"{prefix}.raxml.rfDistances")
|
|
106
|
+
logfile = pathlib.Path(f"{prefix}.raxml.log")
|
|
107
|
+
|
|
108
|
+
# 1. Check if all RAxML-NG files already exist
|
|
109
|
+
files_exist = rfdist.exists() and logfile.exists()
|
|
110
|
+
if not files_exist:
|
|
111
|
+
return False
|
|
112
|
+
|
|
113
|
+
# 2. Run is complete
|
|
114
|
+
run_complete = "Elapsed time:" in logfile.read_text()
|
|
115
|
+
|
|
116
|
+
# 3. Check if the number of pairwise RF-Distance results is correct
|
|
117
|
+
expected_number_of_pairs = n_trees * (n_trees - 1) // 2
|
|
118
|
+
n_pairs_in_file = sum(1 for _ in rfdist.open())
|
|
119
|
+
if n_pairs_in_file != expected_number_of_pairs:
|
|
120
|
+
raise ValueError(
|
|
121
|
+
f"Number of pairwise RF-Distances in {rfdist} ({n_pairs_in_file}) does not match "
|
|
122
|
+
f"the expected number of pairs ({expected_number_of_pairs})."
|
|
123
|
+
f"Please set the `redo` flag to recompute the distances."
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
return files_exist and run_complete
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def rf_distance(
|
|
130
|
+
ml_trees: pathlib.Path,
|
|
131
|
+
prefix: pathlib.Path,
|
|
132
|
+
raxmlng: pathlib.Path,
|
|
133
|
+
n_trees: Optional[int] = None,
|
|
134
|
+
redo: bool = False,
|
|
135
|
+
) -> tuple[int, float]:
|
|
136
|
+
"""
|
|
137
|
+
Compute the number of unique topologies and the average relative RF distance for a set of ML trees.
|
|
138
|
+
If the results already exist, the function will return the results without recomputing them.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
ml_trees (pathlib.Path): Path to the file containing the ML trees.
|
|
142
|
+
prefix (pathlib.Path): Prefix to use for the RAxML-NG output files.
|
|
143
|
+
raxmlng (pathlib.Path): Path to the RAxML-NG executable.
|
|
144
|
+
n_trees (Optional[int]): Number of trees that were inferred.
|
|
145
|
+
If not provided, the number of trees will be inferred from the file.
|
|
146
|
+
Explicitly provide the number of trees if you want to check if existing results for the given
|
|
147
|
+
prefix contain the results for the correct number of trees.
|
|
148
|
+
redo (bool): Flag to redo the computation if the results already exist.
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
tuple[int, float]: The number of unique topologies and the average relative RF distance.
|
|
152
|
+
|
|
153
|
+
"""
|
|
154
|
+
if n_trees is None:
|
|
155
|
+
n_trees = sum(1 for _ in ml_trees.open())
|
|
156
|
+
|
|
157
|
+
if n_trees == 0:
|
|
158
|
+
raise ValueError("At least 1 tree is required.")
|
|
159
|
+
|
|
160
|
+
if n_trees == 1:
|
|
161
|
+
return 1, 0.0
|
|
162
|
+
|
|
163
|
+
if not redo and _rfdist_results_exists_and_correct(prefix, n_trees):
|
|
164
|
+
num_topos, rel_rfdist = get_raxmlng_rfdist_results(
|
|
165
|
+
pathlib.Path(f"{prefix}.raxml.log")
|
|
166
|
+
)
|
|
167
|
+
else:
|
|
168
|
+
raxmlng = RAxMLNG(raxmlng)
|
|
169
|
+
num_topos, rel_rfdist = raxmlng.get_rfdistance_results(
|
|
170
|
+
ml_trees, prefix, **{"redo": None} if redo else {}
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
return num_topos, rel_rfdist
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: PythiaLabelGenerator
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Command line tool to generate the ground-truth phylogenetic difficulty of MSAs
|
|
5
|
+
Project-URL: Homepage, https://github.com/tschuelia/PythiaLabelGenerator
|
|
6
|
+
Author-email: Julia Haag <info@juliaschmid.com>
|
|
7
|
+
License-Expression: GPL-3.0-or-later
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Requires-Python: <3.13,>=3.9
|
|
14
|
+
Requires-Dist: loguru
|
|
15
|
+
Requires-Dist: pythiaphylopredictor>=2.0.0
|
|
16
|
+
Requires-Dist: regex
|
|
17
|
+
Provides-Extra: test
|
|
18
|
+
Requires-Dist: pytest; extra == 'test'
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# Pythia Difficulty Label Generator
|
|
22
|
+
|
|
23
|
+

|
|
24
|
+
|
|
25
|
+
The Pythia Difficulty Label Generator generates the ground-truth phylogenetic difficulty label for an MSA and
|
|
26
|
+
corresponds to the prediction target of our difficulty prediction tool [Pythia](https://github.com/tschuelia/PyPythia).
|
|
27
|
+
|
|
28
|
+
> [!CAUTION]
|
|
29
|
+
> Computing the ground-truth difficulty for an MSA is very time-consuming and requires a lot of computational resources,
|
|
30
|
+
> especially for MSAs with many sites and/or taxa.
|
|
31
|
+
>
|
|
32
|
+
> Please use our (very accurate) difficulty prediction tool [Pythia](https://github.com/tschuelia/PyPythia) instead
|
|
33
|
+
> whenever possible.
|
|
34
|
+
|
|
35
|
+
The ground-truth difficulty is computed according to our definition published in [Haag _et
|
|
36
|
+
al._ (2022)](https://doi.org/10.1093/molbev/msac254):
|
|
37
|
+
|
|
38
|
+
Let $`N_{\text{all}}`$ be the number of inferred ML trees.
|
|
39
|
+
We first compute the average pairwise relative Robinson-Foulds (RF) distance between all trees ($`RF_{\text{all}}`$), as
|
|
40
|
+
well as the number of unique tree topologies among the inferred trees ($`N^*_{\text{all}}`$).
|
|
41
|
+
We filter the inferred trees using likelihood-based statistical tests to obtain the set of $`N_{\text{pl}}`$ _plausible
|
|
42
|
+
trees_.
|
|
43
|
+
We again compute the average pairwise RF distance between the plausible trees ($`RF_{\text{pl}}`$) and the number of
|
|
44
|
+
unique tree topologies among the plausible trees ($`N^*_{\text{pl}}`$).
|
|
45
|
+
|
|
46
|
+
The difficulty is then computed as follows:
|
|
47
|
+
|
|
48
|
+
```math
|
|
49
|
+
\text{difficulty} = \frac{1}{5} \cdot \bigg[ RF_{\text{all}} + RF_{\text{pl}}
|
|
50
|
+
+ \frac{N^*_{\text{all}}}{N_{\text{all}}} + \frac{N^*_{\text{pl}}}{N_{\text{pl}}}
|
|
51
|
+
+ \left( 1 - \frac{N_{\text{pl}}}{N_{\text{all}}} \right) \bigg]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
For further details on the reasoning and validation of this difficulty, please refer to the publication linked above.
|
|
55
|
+
|
|
56
|
+
We infer the ML trees using [RAxML-NG](https://github.com/amkozlov/raxml-ng) and use the statistical significance tests
|
|
57
|
+
as implemented in [IQ-TREE](http://www.iqtree.org).
|
|
58
|
+
Per default, the difficulty is based on $`N_{\text{all}}=100`$ ML trees.
|
|
59
|
+
Note that this number can be adjusted by the user, however, the difficulty will only be an approximation if the number
|
|
60
|
+
of trees is changed.
|
|
61
|
+
|
|
62
|
+
## Prediction of Phylogenetic Difficulty
|
|
63
|
+
|
|
64
|
+
As stated above, computing the ground-truth difficulty for an MSA is very time-consuming and requires a lot of
|
|
65
|
+
computational resources, especially for MSAs with many sites and/or taxa.
|
|
66
|
+
Please use our (very accurate) difficulty prediction tool [Pythia](https://github.com/tschuelia/PyPythia) instead
|
|
67
|
+
whenever possible.
|
|
68
|
+
For instance, inferring _a single ML tree_ for an MSA
|
|
69
|
+
comprising [SARS-CoV-2 sequences](https://doi.org/10.1093/molbev/msaa314) (approx. 5k taxa and 28.5k sites) takes about
|
|
70
|
+
12 hours on a large compute cluster. Using Pythia instead, we can predict the same MSA to be very difficult in about 2.5
|
|
71
|
+
minutes on a standard MacBook.
|
|
72
|
+
|
|
73
|
+
Only use this tool if you need the ground-truth difficulty for a specific MSA and you are sure that Pythia is unable to
|
|
74
|
+
predict the difficulty accurately.
|
|
75
|
+
The only case where we observed Pythia to fail is for language MSAs, so if you are working with DNA, Protein, or
|
|
76
|
+
biological morphological data, Pythia should work just fine 😉
|
|
77
|
+
|
|
78
|
+
## Installation
|
|
79
|
+
|
|
80
|
+
#### Requirements
|
|
81
|
+
|
|
82
|
+
To use this labelling tool, you need to install
|
|
83
|
+
|
|
84
|
+
- RAxML-NG: See [the RAxML-NG GitHub repository](https://github.com/amkozlov/raxml-ng) for installation instructions.
|
|
85
|
+
Please make sure that you install a RAxML-NG version < 2.
|
|
86
|
+
- IQ-TREE: See [the IQ-TREE website](http://www.iqtree.org) for installation instructions. Please install IQ-TREE
|
|
87
|
+
version 2 or higher.
|
|
88
|
+
|
|
89
|
+
#### Install via conda (recommended)
|
|
90
|
+
|
|
91
|
+
This package will soon be available on conda-forge :)
|
|
92
|
+
|
|
93
|
+
#### Install using pip
|
|
94
|
+
|
|
95
|
+
You can install the package using pip:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
pip install pythialabelgenerator
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Usage
|
|
102
|
+
|
|
103
|
+
This label-generator is primarily a command line tool. You can call it using the `label` command, for instance, to
|
|
104
|
+
compute the difficulty for the example MSA provided in the `examples` directory run
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
label -m examples/example.phy
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
This will infer 100 ML trees using RAxML-NG and run the statistical tests using IQ-TREE. The difficulty will be printed
|
|
111
|
+
to the console.
|
|
112
|
+
The output will look something like this:
|
|
113
|
+
|
|
114
|
+
```text
|
|
115
|
+
Difficulty LabelGenerator version 1.0.0 released by The Exelixis Lab
|
|
116
|
+
Developed by: Julia Haag
|
|
117
|
+
Latest version: https://github.com/tschuelia/LabelGenerator
|
|
118
|
+
Questions/problems/suggestions? Please open an issue on GitHub.
|
|
119
|
+
|
|
120
|
+
LabelGenerator was called at 06-Mar-2025 15:15:03 as follows:
|
|
121
|
+
|
|
122
|
+
label -m examples/example.phy
|
|
123
|
+
|
|
124
|
+
[00:00:00] Starting label computation.
|
|
125
|
+
[00:00:00] Inferring 100 ML trees using RAxML-NG.
|
|
126
|
+
[00:00:23] Computing RF-Distance between ML trees.
|
|
127
|
+
[00:00:23] > RF-Distance ML trees: 0.78
|
|
128
|
+
[00:00:23] > Unique topologies ML trees: 100
|
|
129
|
+
[00:00:23] Running IQ-TREE statistical tests.
|
|
130
|
+
[00:00:27] Filtering plausible ML trees.
|
|
131
|
+
[00:00:27] > Found 18 plausible trees.
|
|
132
|
+
[00:00:27] Computing RF-Distance between plausible ML trees.
|
|
133
|
+
[00:00:27] > RF-Distance plausible trees: 0.78
|
|
134
|
+
[00:00:27] > Unique topologies plausible trees: 18
|
|
135
|
+
|
|
136
|
+
Ground Truth Difficulty for examples/example.phy: 0.875
|
|
137
|
+
|
|
138
|
+
Total runtime: 27.42 seconds.
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Depending on your system setup, you might need to pass a RAxML-NG and IQ-TREE binary path to the label generator.
|
|
142
|
+
You can do this using the `-r` and `-i` options, respectively. This is required in case `raxml-ng` and/or `iqtree2` are
|
|
143
|
+
not in your `$PATH`.
|
|
144
|
+
|
|
145
|
+
Note that this `examply.phy` MSA is not the same exemplary MSA as we provide in the PyPythia repository, so please don't compare this ground-truth lable to the exemplary prediction in PyPythia 😉
|
|
146
|
+
|
|
147
|
+
For a full list of command line options, run `label -h`:
|
|
148
|
+
|
|
149
|
+
```text
|
|
150
|
+
Difficulty LabelGenerator version 1.0.0 released by The Exelixis Lab
|
|
151
|
+
Developed by: Julia Haag
|
|
152
|
+
Latest version: https://github.com/tschuelia/LabelGenerator
|
|
153
|
+
Questions/problems/suggestions? Please open an issue on GitHub.
|
|
154
|
+
|
|
155
|
+
usage: label [-h] -m MSA -r RAXMLNG -i IQTREE [-t THREADS] [-s SEED] [-p PREFIX] [--model MODEL] [--ntrees NTREES] [--redo] [-V]
|
|
156
|
+
|
|
157
|
+
Generate the ground truth difficulty for the given MSA.
|
|
158
|
+
|
|
159
|
+
options:
|
|
160
|
+
-h, --help show this help message and exit
|
|
161
|
+
-m MSA, --msa MSA Multiple Sequence Alignment to compute the ground truth difficulty for. Must be in either phylip or fasta format.
|
|
162
|
+
-r RAXMLNG, --raxmlng RAXMLNG
|
|
163
|
+
Path to the binary of RAxML-NG. For install instructions see https://github.com/amkozlov/raxml-ng.(default: 'raxml-
|
|
164
|
+
ng' if in $PATH, otherwise this option is mandatory).
|
|
165
|
+
-i IQTREE, --iqtree IQTREE
|
|
166
|
+
Path to the binary of IQ-TREE2. For install instructions see http://www.iqtree.org.(default: 'iqtree2' if in $PATH,
|
|
167
|
+
otherwise this option is mandatory).
|
|
168
|
+
-t THREADS, --threads THREADS
|
|
169
|
+
Number of threads to use for the RAxML-NG tree inference and IQ-TREE statistical tests (default: autoconfig in RAxML-NG and IQ-TREE).
|
|
170
|
+
-s SEED, --seed SEED Seed for the RAxML-NG tree inference (default: 0).
|
|
171
|
+
-p PREFIX, --prefix PREFIX
|
|
172
|
+
Prefix of the RAxML-NG and IQ-TREE log and result files (default: MSA file name).
|
|
173
|
+
--model MODEL Model to use for the RAxML-NG tree inference (default: 'GTR+G' for DNA, 'LG+G' for AA, and 'MULTIx_GTR' for
|
|
174
|
+
morphological data where x is the maximum state value in the MSA).
|
|
175
|
+
--ntrees NTREES Number of ML trees to infer (default: 100)
|
|
176
|
+
--redo Redo all computations, even if the results already exist.
|
|
177
|
+
-V, --version Print the version number and exit.
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Please note that inferring 100 ML trees for a large MSA can take a long time. You can adjust the number of trees to
|
|
181
|
+
infer using the `--ntrees` option. However, using fewer than 100 trees will likely result in slight different
|
|
182
|
+
difficulties, as the difficulty is based on the average pairwise RF distance between the inferred trees.
|
|
183
|
+
|
|
184
|
+
### Result Files
|
|
185
|
+
|
|
186
|
+
Running this labelling tool will result in the following files:
|
|
187
|
+
|
|
188
|
+
- `{prefix}.raxml.*`: RAxML-NG log and result files.
|
|
189
|
+
- `{prefix}.iqtree.*`: IQ-TREE log and result files.
|
|
190
|
+
- `{prefix}.labelGen.log`: Log file containing the output of the label generator. This is the same output as printed to
|
|
191
|
+
the terminal.
|
|
192
|
+
|
|
193
|
+
You can set the prefix of these files using the `-p` option. By default, the prefix is the name of the MSA file.
|
|
194
|
+
Note that RAxML-NG and IQ-TREE refuse to overwrite existing files. If you want to redo the computations, you can use the
|
|
195
|
+
`--redo` option.
|
|
196
|
+
Please also specify the `--redo` option if you want to change the number of trees to infer using the `--ntrees` option
|
|
197
|
+
for the same prefix. Otherwise,
|
|
198
|
+
the label generator will exit with an error message.
|
|
199
|
+
|
|
200
|
+
### Input Data
|
|
201
|
+
|
|
202
|
+
You can provide the MSA in either phylip or fasta format. We currently support DNA, AA, and categorical data.
|
|
203
|
+
The label generator will automatically determine the data type and select an appropriate model for tree inference and
|
|
204
|
+
statistical tests.
|
|
205
|
+
For DNA data, we use the `GTR+G` model, for AA data the `LG+G` model, and for categorical data the `MULTIx_GTR` model, where `x`
|
|
206
|
+
is the maximum state value in the MSA.
|
|
207
|
+
We used these models to generate the ground-truth labels for training Pythia. If you want to specify a different model,
|
|
208
|
+
you can do so using the `--model` option.
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
## Citation
|
|
212
|
+
We will soon publish a pre-print on bioRxiv with updates on our Pythia difficulty prediction tool that will also include a
|
|
213
|
+
brief description of this new labelling tool. Please cite this pre-print if you use this tool in your research.
|
|
214
|
+
|
|
215
|
+
The link to the paper will be added soon 🙂
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
labelgenerator/__init__.py,sha256=kwlgs69Sq0mro8LPwa_xG4g-d_Z6hngg04IHgvkC8HU,374
|
|
2
|
+
labelgenerator/iqtree.py,sha256=tyEFM54HzvkgUrPTAPoudEDxBrDw1dD2vkym6CPDBss,4444
|
|
3
|
+
labelgenerator/iqtree_parser.py,sha256=bZuVMRzlBsaEi0rIAEWuILc_bkahBkEW2r7i4mKtPcw,6485
|
|
4
|
+
labelgenerator/label.py,sha256=6L7s_xDYhtX7kNvx9UVf5e6KYSuZBBd6ohJHEUSODZ0,8699
|
|
5
|
+
labelgenerator/logger.py,sha256=XbYSJyL-WsjjaObI6hS4zM8E_GzqlVRvBbXtcFJcOsE,861
|
|
6
|
+
labelgenerator/main.py,sha256=nXmxEDL_i_IAiRmS0c9Z96F4IORAlHpjypQtzGwGW-M,4812
|
|
7
|
+
labelgenerator/raxmlng.py,sha256=HwK3kPtBeUlsIkXI10T792cJ0YRajdeJ42VcTOMK8lQ,6075
|
|
8
|
+
pythialabelgenerator-1.0.0.dist-info/METADATA,sha256=ZP94OQJ_j_SEsOwoLRxN-BVfQqyNVDrfj89MWpsH4vI,10438
|
|
9
|
+
pythialabelgenerator-1.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
10
|
+
pythialabelgenerator-1.0.0.dist-info/entry_points.txt,sha256=sI1fBhWG20vN_beYNPpLY7Ex6tXIo0ybBhgj7xTuslM,51
|
|
11
|
+
pythialabelgenerator-1.0.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
12
|
+
pythialabelgenerator-1.0.0.dist-info/RECORD,,
|