cdhit-reader 0.3.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cdhit_reader-0.3.0/cdhit_reader.egg-info → cdhit_reader-0.5.0}/PKG-INFO +19 -1
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/README.md +18 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/__init__.py +2 -1
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/_cli.py +15 -16
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/_compare.py +48 -42
- cdhit_reader-0.5.0/cdhit_reader/_compare_otus.py +332 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/_fasta.py +15 -20
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/_reader.py +65 -143
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/_version.py +1 -1
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/test/test_clstr.py +95 -12
- cdhit_reader-0.5.0/cdhit_reader/test/test_clustering.py +66 -0
- cdhit_reader-0.5.0/cdhit_reader/test/test_compare.py +10 -0
- cdhit_reader-0.5.0/cdhit_reader/test/test_compare_otus.py +130 -0
- cdhit_reader-0.5.0/cdhit_reader/test/test_examples.py +44 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/test/test_fasta.py +24 -0
- cdhit_reader-0.5.0/cdhit_reader/test/test_integration.py +30 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0/cdhit_reader.egg-info}/PKG-INFO +19 -1
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader.egg-info/SOURCES.txt +6 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader.egg-info/entry_points.txt +1 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/setup.py +1 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/LICENSE.md +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/MANIFEST.in +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/_testit.py +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/test/__init__.py +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/test/aa.clstr +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/test/nt.clstr +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader/test/test_package_import.py +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader.egg-info/dependency_links.txt +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader.egg-info/requires.txt +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader.egg-info/top_level.txt +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/cdhit_reader.egg-info/zip-safe +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/pyproject.toml +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/setup.cfg +0 -0
- {cdhit_reader-0.3.0 → cdhit_reader-0.5.0}/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cdhit-reader
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: CD-HIT cluster parser
|
|
5
5
|
Home-page: https://github.com/telatin/cdhit-parser
|
|
6
6
|
Download-URL: https://github.com/telatin/cdhit-parser
|
|
@@ -60,6 +60,24 @@ Load all clusters in to a list:
|
|
|
60
60
|
clusters = read_cdhit(input).read_items()
|
|
61
61
|
```
|
|
62
62
|
|
|
63
|
+
### Sequence to cluster lookup
|
|
64
|
+
|
|
65
|
+
`Clustering` loads a whole .clstr file and provides a reverse index
|
|
66
|
+
(`seqcluster`) mapping each sequence name to the name of its cluster:
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from cdhit_reader import Clustering
|
|
70
|
+
|
|
71
|
+
clustering = Clustering.from_file(input)
|
|
72
|
+
|
|
73
|
+
print(len(clustering)) # number of clusters
|
|
74
|
+
for cluster in clustering: # iterate over the clusters
|
|
75
|
+
print(cluster.name)
|
|
76
|
+
|
|
77
|
+
# Which cluster does a sequence belong to?
|
|
78
|
+
print(clustering.seqcluster["seq1.A"]) # e.g. "Cluster 0"
|
|
79
|
+
```
|
|
80
|
+
|
|
63
81
|
## Read FASTA file
|
|
64
82
|
|
|
65
83
|
```python
|
|
@@ -29,6 +29,24 @@ Load all clusters in to a list:
|
|
|
29
29
|
clusters = read_cdhit(input).read_items()
|
|
30
30
|
```
|
|
31
31
|
|
|
32
|
+
### Sequence to cluster lookup
|
|
33
|
+
|
|
34
|
+
`Clustering` loads a whole .clstr file and provides a reverse index
|
|
35
|
+
(`seqcluster`) mapping each sequence name to the name of its cluster:
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from cdhit_reader import Clustering
|
|
39
|
+
|
|
40
|
+
clustering = Clustering.from_file(input)
|
|
41
|
+
|
|
42
|
+
print(len(clustering)) # number of clusters
|
|
43
|
+
for cluster in clustering: # iterate over the clusters
|
|
44
|
+
print(cluster.name)
|
|
45
|
+
|
|
46
|
+
# Which cluster does a sequence belong to?
|
|
47
|
+
print(clustering.seqcluster["seq1.A"]) # e.g. "Cluster 0"
|
|
48
|
+
```
|
|
49
|
+
|
|
32
50
|
## Read FASTA file
|
|
33
51
|
|
|
34
52
|
```python
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from importlib import import_module
|
|
2
2
|
|
|
3
|
-
from ._reader import ParsingError, ClusterSequence, Cluster, ClstrReader, read_cdhit, SeqType, Strand
|
|
3
|
+
from ._reader import ParsingError, ClusterSequence, Cluster, Clustering, ClstrReader, read_cdhit, SeqType, Strand
|
|
4
4
|
from ._fasta import FastaParsingError, Sequence, FastaReader, read_fasta
|
|
5
5
|
from ._version import __version__
|
|
6
6
|
|
|
@@ -15,6 +15,7 @@ __all__ = [
|
|
|
15
15
|
"FastaParsingError",
|
|
16
16
|
"ClusterSequence",
|
|
17
17
|
"Cluster",
|
|
18
|
+
"Clustering",
|
|
18
19
|
"ClstrReader",
|
|
19
20
|
"read_cdhit",
|
|
20
21
|
"FastaReader",
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
import sys
|
|
3
1
|
from statistics import mean
|
|
4
2
|
|
|
5
3
|
import click
|
|
6
4
|
|
|
7
5
|
from ._reader import read_cdhit
|
|
8
6
|
from ._version import __version__
|
|
9
|
-
#from ._writer import write_fasta
|
|
10
7
|
|
|
11
8
|
|
|
12
9
|
@click.command(context_settings=dict(help_option_names=["-h", "--help"]))
|
|
@@ -22,31 +19,33 @@ def cli(clstr, stats: bool, hist: bool, all: bool):
|
|
|
22
19
|
\b
|
|
23
20
|
Warning
|
|
24
21
|
-------
|
|
25
|
-
The commad line interface is in EXPERIMENTAL stage.
|
|
22
|
+
The commad line interface is in EXPERIMENTAL stage.
|
|
26
23
|
"""
|
|
27
24
|
|
|
28
25
|
nitems = 0
|
|
29
26
|
nseqs = 0
|
|
30
27
|
seq_lens = []
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
28
|
+
with read_cdhit(clstr) as reader:
|
|
29
|
+
for item in reader:
|
|
30
|
+
seq_lens.append(len(item))
|
|
31
|
+
nitems += 1
|
|
32
|
+
nseqs += len(item)
|
|
33
|
+
if all:
|
|
34
|
+
print(item)
|
|
35
|
+
for s in item.sequences:
|
|
36
|
+
print(f" {s}")
|
|
39
37
|
if stats:
|
|
40
38
|
click.echo(f"Input file: {clstr}")
|
|
41
39
|
click.echo(f"Number of clusters: {nitems}")
|
|
42
40
|
click.echo(f"Total sequences: {nseqs}")
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
41
|
+
|
|
42
|
+
if seq_lens:
|
|
43
|
+
msg = f"Cluster size: min {min(seq_lens)}, mean {mean(seq_lens):.2f}, max {max(seq_lens)}"
|
|
44
|
+
click.echo(msg)
|
|
46
45
|
|
|
47
46
|
if hist:
|
|
48
47
|
show_hist(seq_lens)
|
|
49
48
|
|
|
50
49
|
|
|
51
50
|
def show_hist(seq_lens):
|
|
52
|
-
pass
|
|
51
|
+
pass
|
|
@@ -1,26 +1,35 @@
|
|
|
1
|
-
from email.policy import default
|
|
2
1
|
import sys
|
|
3
|
-
|
|
2
|
+
import os
|
|
3
|
+
import subprocess
|
|
4
|
+
import tempfile
|
|
4
5
|
|
|
5
6
|
import click
|
|
7
|
+
from xopen import xopen
|
|
6
8
|
|
|
7
9
|
from ._reader import read_cdhit
|
|
8
10
|
from ._version import __version__
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
|
|
12
|
+
TAG1 = "1:::"
|
|
13
|
+
TAG2 = "2:::"
|
|
14
|
+
|
|
13
15
|
|
|
14
16
|
def has_cdhit():
|
|
15
17
|
cmd = ["cd-hit", "-h"]
|
|
16
|
-
# Run cmd and
|
|
18
|
+
# Run cmd and inspect stdout, ignore exit status
|
|
17
19
|
try:
|
|
18
20
|
cdout = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.decode("utf-8")
|
|
19
|
-
|
|
20
|
-
return True
|
|
21
|
+
return "CD-HIT" in cdout
|
|
21
22
|
except Exception:
|
|
22
23
|
return False
|
|
23
|
-
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def strip_tag(name: str) -> str:
|
|
27
|
+
"""
|
|
28
|
+
Remove the relabeling prefix (TAG1/TAG2, same length) from a sequence name.
|
|
29
|
+
"""
|
|
30
|
+
return name[len(TAG1):]
|
|
31
|
+
|
|
32
|
+
|
|
24
33
|
def read_fasta(path):
|
|
25
34
|
name = None
|
|
26
35
|
comment = ""
|
|
@@ -38,11 +47,13 @@ def read_fasta(path):
|
|
|
38
47
|
seq += line
|
|
39
48
|
yield name, comment, seq
|
|
40
49
|
|
|
50
|
+
|
|
41
51
|
def relabel_fasta(path, prefix, outpath):
|
|
42
52
|
with xopen(outpath, "a") as out:
|
|
43
53
|
for name, comment, seq in read_fasta(path):
|
|
44
54
|
out.write(">{}{}{}{}\n{}\n".format(prefix, name, " " if len(comment) > 1 else "", comment, seq))
|
|
45
55
|
|
|
56
|
+
|
|
46
57
|
def split_cluster(cluster, tag1, tag2):
|
|
47
58
|
pool1, pool2 = [], []
|
|
48
59
|
for sequence in cluster.sequences:
|
|
@@ -52,17 +63,19 @@ def split_cluster(cluster, tag1, tag2):
|
|
|
52
63
|
pool2.append(sequence)
|
|
53
64
|
else:
|
|
54
65
|
raise ValueError("Sequence {} does not start with {} or {}".format(sequence.name, tag1, tag2))
|
|
66
|
+
|
|
67
|
+
|
|
55
68
|
@click.command(context_settings=dict(help_option_names=["-h", "--help"]))
|
|
56
69
|
@click.version_option(__version__)
|
|
57
70
|
@click.argument("fasta1", type=click.Path(exists=True))
|
|
58
71
|
@click.argument("fasta2", type=click.Path(exists=True))
|
|
59
|
-
@click.option("--tag1",
|
|
60
|
-
@click.option("--tag2",
|
|
72
|
+
@click.option("--tag1", help="Name of the first dataset")
|
|
73
|
+
@click.option("--tag2", help="Name of the second dataset")
|
|
61
74
|
@click.option("--id", help="Identity threshold [default: 0.9]", default=0.95, type=float)
|
|
62
75
|
@click.option("--type", type=click.STRING, help="Type of the sequences (nucl or prot)")
|
|
63
|
-
@click.option("--tempdir",type=click.Path(exists=True), help="Temporary directory for intermediate files", default=tempfile.gettempdir())
|
|
76
|
+
@click.option("--tempdir", type=click.Path(exists=True), help="Temporary directory for intermediate files", default=tempfile.gettempdir())
|
|
64
77
|
@click.option("--verbose", default=False, is_flag=True, help="Show verbose information")
|
|
65
|
-
def compare(fasta1, fasta2, tag1: str, tag2: str, tempdir, type: str, id: float,verbose: bool):
|
|
78
|
+
def compare(fasta1, fasta2, tag1: str, tag2: str, tempdir, type: str, id: float, verbose: bool):
|
|
66
79
|
"""
|
|
67
80
|
Compare FASTA files
|
|
68
81
|
|
|
@@ -75,12 +88,9 @@ def compare(fasta1, fasta2, tag1: str, tag2: str, tempdir, type: str, id: float,
|
|
|
75
88
|
click.echo("cd-hit is not installed. Please install it and try again.")
|
|
76
89
|
sys.exit(1)
|
|
77
90
|
# Generate temporary directory inside "tempdir"
|
|
78
|
-
TAG1 = "1:::"
|
|
79
|
-
TAG2 = "2:::"
|
|
80
91
|
# tmp is a temporary directory not to be deleted
|
|
81
92
|
tmp = tempfile.mkdtemp(dir=tempdir, prefix="cdhit_")
|
|
82
93
|
|
|
83
|
-
#tmp = tempfile.TemporaryDirectory(dir=tempdir, prefix="cdhit_reader_", suffix=".tmp")
|
|
84
94
|
if verbose:
|
|
85
95
|
print("Temporary directory: {}".format(tmp), file=sys.stderr)
|
|
86
96
|
|
|
@@ -97,42 +107,41 @@ def compare(fasta1, fasta2, tag1: str, tag2: str, tempdir, type: str, id: float,
|
|
|
97
107
|
if prefix1 == prefix2:
|
|
98
108
|
print("Warning: prefixes are identical ({}): specify manual prefixes with --tag1 and --tag2".format(prefix1))
|
|
99
109
|
exit(1)
|
|
100
|
-
|
|
110
|
+
|
|
101
111
|
# Delete fasta_file if present:
|
|
102
112
|
if os.path.exists(fasta_file):
|
|
103
113
|
os.remove(fasta_file)
|
|
104
|
-
|
|
114
|
+
|
|
105
115
|
if verbose:
|
|
106
116
|
print("Relabeling {} to {}".format(fasta1, fasta_file), file=sys.stderr)
|
|
107
|
-
|
|
117
|
+
|
|
108
118
|
relabel_fasta(fasta1, TAG1, fasta_file)
|
|
109
119
|
|
|
110
120
|
if verbose:
|
|
111
121
|
print("Relabeling {} to {}".format(fasta2, fasta_file), file=sys.stderr)
|
|
112
|
-
|
|
122
|
+
|
|
113
123
|
relabel_fasta(fasta2, TAG2, fasta_file)
|
|
114
124
|
|
|
115
125
|
tags = {
|
|
116
126
|
TAG1: prefix1,
|
|
117
127
|
TAG2: prefix2
|
|
118
|
-
}
|
|
128
|
+
}
|
|
119
129
|
if verbose:
|
|
120
130
|
print("Relabeling {} to {} (prefix: {})".format(fasta1, fasta_file, prefix1), file=sys.stderr)
|
|
121
131
|
print("Relabeling {} to {} (prefix: {})".format(fasta2, fasta_file, prefix2), file=sys.stderr)
|
|
122
|
-
|
|
123
|
-
cmd = ["cd-hit" if type=="prot" else "cd-hit-est", "-i", fasta_file, "-o", clstr_file, "-c", str(id), "-d", "1000"]
|
|
132
|
+
|
|
133
|
+
cmd = ["cd-hit" if type == "prot" else "cd-hit-est", "-i", fasta_file, "-o", clstr_file, "-c", str(id), "-d", "1000"]
|
|
124
134
|
if verbose:
|
|
125
135
|
print("Running {}".format(" ".join(cmd)), file=sys.stderr)
|
|
126
136
|
subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
127
137
|
|
|
128
|
-
|
|
129
|
-
stats = {TAG1: [], TAG2: [], "both": [], "multi": [], "dupl_" + TAG1: [], "dupl_" + TAG2: []}
|
|
138
|
+
stats = {TAG1: [], TAG2: [], "both": [], "multi": [], "dupl_" + TAG1: [], "dupl_" + TAG2: []}
|
|
130
139
|
for cluster in read_cdhit(clstr_file + ".clstr"):
|
|
131
|
-
|
|
132
|
-
pool =
|
|
140
|
+
|
|
141
|
+
pool = cluster.refname[:len(TAG1)]
|
|
133
142
|
if pool != TAG1 and pool != TAG2:
|
|
134
143
|
raise ValueError("Cluster {} does not start with {} or {}".format(cluster.refname, TAG1, TAG2))
|
|
135
|
-
seqname = cluster.refname
|
|
144
|
+
seqname = strip_tag(cluster.refname)
|
|
136
145
|
if len(cluster) == 1:
|
|
137
146
|
# Singleton
|
|
138
147
|
stats[pool].append(seqname)
|
|
@@ -141,31 +150,28 @@ def compare(fasta1, fasta2, tag1: str, tag2: str, tempdir, type: str, id: float,
|
|
|
141
150
|
pair = []
|
|
142
151
|
check = False
|
|
143
152
|
for seq in cluster.sequences:
|
|
144
|
-
sub_pool =
|
|
145
|
-
sub_seqname = (seq.name)
|
|
153
|
+
sub_pool = seq.name[:len(TAG1)]
|
|
154
|
+
sub_seqname = strip_tag(seq.name)
|
|
146
155
|
pair.append(sub_seqname)
|
|
147
156
|
if sub_pool != pool:
|
|
148
157
|
check = True
|
|
149
|
-
if check
|
|
158
|
+
if check:
|
|
150
159
|
stats["both"].append(":".join(pair))
|
|
151
160
|
else:
|
|
152
|
-
stats["dupl_" + pool
|
|
161
|
+
stats["dupl_" + pool].append(":".join(pair))
|
|
153
162
|
else:
|
|
154
|
-
|
|
163
|
+
|
|
155
164
|
stats["multi"].append(",".join(i.name for i in cluster.sequences))
|
|
156
|
-
|
|
157
165
|
|
|
158
|
-
for key,
|
|
159
|
-
print("{} {}".format(key, len(
|
|
160
|
-
for seqnames in
|
|
166
|
+
for key, seqnames_list in stats.items():
|
|
167
|
+
print("{} {}".format(key, len(seqnames_list)), file=sys.stderr)
|
|
168
|
+
for seqnames in seqnames_list:
|
|
161
169
|
key = tags[key] if key in tags else key
|
|
162
|
-
key = "dupl_" + tags[key[-1*len(TAG1):]] if key[-1*len(TAG1):] in tags else key
|
|
170
|
+
key = "dupl_" + tags[key[-1 * len(TAG1):]] if key[-1 * len(TAG1):] in tags else key
|
|
163
171
|
seqnames = seqnames.replace(TAG1, tags[TAG1] + "#").replace(TAG2, tags[TAG2] + "#")
|
|
164
|
-
|
|
172
|
+
|
|
165
173
|
print(key, seqnames, sep="\t")
|
|
166
174
|
|
|
167
|
-
#if not verbose:
|
|
168
|
-
# os.remove(tmp)
|
|
169
175
|
|
|
170
176
|
def show_hist(seq_lens):
|
|
171
177
|
pass
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Dict, Iterable, List
|
|
6
|
+
import csv
|
|
7
|
+
import os
|
|
8
|
+
import shutil
|
|
9
|
+
import subprocess
|
|
10
|
+
import tempfile
|
|
11
|
+
|
|
12
|
+
import click
|
|
13
|
+
|
|
14
|
+
from ._fasta import Sequence, read_fasta
|
|
15
|
+
from ._reader import Cluster, read_cdhit
|
|
16
|
+
from ._version import __version__
|
|
17
|
+
|
|
18
|
+
COMPRESSED_SUFFIXES = (".gz", ".bz2", ".xz", ".zst")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class RenamedSequence:
|
|
23
|
+
source_label: str
|
|
24
|
+
renamed_name: str
|
|
25
|
+
original: Sequence
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class CategorySummary:
|
|
30
|
+
label: str
|
|
31
|
+
description: str
|
|
32
|
+
output_path: Path
|
|
33
|
+
clusters: int = 0
|
|
34
|
+
output_records: int = 0
|
|
35
|
+
source1_members: int = 0
|
|
36
|
+
source2_members: int = 0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def has_cdhit_est() -> bool:
|
|
40
|
+
return shutil.which("cd-hit-est") is not None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def input_prefix(path: Path) -> str:
|
|
44
|
+
"""
|
|
45
|
+
Return a safe prefix derived from the input filename.
|
|
46
|
+
"""
|
|
47
|
+
name = path.name
|
|
48
|
+
for suffix in COMPRESSED_SUFFIXES:
|
|
49
|
+
if name.endswith(suffix):
|
|
50
|
+
name = name[: -len(suffix)]
|
|
51
|
+
break
|
|
52
|
+
|
|
53
|
+
stem = Path(name).stem
|
|
54
|
+
return stem.replace("___", "__")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _record_name(prefix: str, seq_name: str) -> str:
|
|
58
|
+
return f"{prefix}___{seq_name}"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def write_relabeled_fasta(input_path: Path, prefix: str, output_path: Path, append: bool) -> Dict[str, RenamedSequence]:
|
|
62
|
+
"""
|
|
63
|
+
Relabel a FASTA file and append it to the concatenated input used for CD-HIT-EST.
|
|
64
|
+
"""
|
|
65
|
+
mode = "a" if append else "w"
|
|
66
|
+
renamed: Dict[str, RenamedSequence] = {}
|
|
67
|
+
|
|
68
|
+
with output_path.open(mode, encoding="utf-8") as handle:
|
|
69
|
+
for seq in read_fasta(input_path):
|
|
70
|
+
renamed_name = _record_name(prefix, seq.name)
|
|
71
|
+
if renamed_name in renamed:
|
|
72
|
+
raise click.ClickException(
|
|
73
|
+
f"Duplicate sequence identifier after relabeling: {renamed_name}"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
renamed[renamed_name] = RenamedSequence(prefix, renamed_name, seq)
|
|
77
|
+
handle.write(f">{renamed_name}\n{seq.sequence}\n")
|
|
78
|
+
|
|
79
|
+
return renamed
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def classify_cluster(cluster: Cluster, prefix1: str, prefix2: str) -> tuple[str, int, int]:
|
|
83
|
+
prefix1_marker = f"{prefix1}___"
|
|
84
|
+
prefix2_marker = f"{prefix2}___"
|
|
85
|
+
count1 = 0
|
|
86
|
+
count2 = 0
|
|
87
|
+
|
|
88
|
+
for member in cluster.sequences:
|
|
89
|
+
if member.name.startswith(prefix1_marker):
|
|
90
|
+
count1 += 1
|
|
91
|
+
elif member.name.startswith(prefix2_marker):
|
|
92
|
+
count2 += 1
|
|
93
|
+
else:
|
|
94
|
+
raise click.ClickException(
|
|
95
|
+
f"Cluster member {member.name!r} does not belong to either input dataset."
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
if count1 == 1 and count2 == 1 and len(cluster) == 2:
|
|
99
|
+
return "one_to_one_matches", count1, count2
|
|
100
|
+
if count1 > 0 and count2 > 0:
|
|
101
|
+
return "shared_multi", count1, count2
|
|
102
|
+
if count1 > 0:
|
|
103
|
+
return "only_file1", count1, count2
|
|
104
|
+
return "only_file2", count1, count2
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def sequences_for_cluster(
|
|
108
|
+
cluster: Cluster,
|
|
109
|
+
renamed_sequences: Dict[str, RenamedSequence],
|
|
110
|
+
all_members: bool,
|
|
111
|
+
) -> List[Sequence]:
|
|
112
|
+
if all_members:
|
|
113
|
+
sequence_names = [member.name for member in cluster.sequences]
|
|
114
|
+
else:
|
|
115
|
+
if cluster.refname is None:
|
|
116
|
+
raise click.ClickException(f"Cluster {cluster.name!r} does not define a representative sequence.")
|
|
117
|
+
sequence_names = [cluster.refname]
|
|
118
|
+
|
|
119
|
+
sequences: List[Sequence] = []
|
|
120
|
+
for renamed_name in sequence_names:
|
|
121
|
+
try:
|
|
122
|
+
sequences.append(renamed_sequences[renamed_name].original)
|
|
123
|
+
except KeyError as exc:
|
|
124
|
+
raise click.ClickException(
|
|
125
|
+
f"Could not restore original sequence for renamed identifier {renamed_name!r}."
|
|
126
|
+
) from exc
|
|
127
|
+
return sequences
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def write_fasta_output(path: Path, sequences: Iterable[Sequence]) -> int:
|
|
131
|
+
count = 0
|
|
132
|
+
with path.open("w", encoding="utf-8") as handle:
|
|
133
|
+
for sequence in sequences:
|
|
134
|
+
handle.write(f"{sequence}\n")
|
|
135
|
+
count += 1
|
|
136
|
+
return count
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def write_stats(path: Path, summaries: Iterable[CategorySummary]) -> None:
|
|
140
|
+
with path.open("w", encoding="utf-8", newline="") as handle:
|
|
141
|
+
writer = csv.writer(handle, delimiter="\t")
|
|
142
|
+
writer.writerow(
|
|
143
|
+
[
|
|
144
|
+
"category",
|
|
145
|
+
"description",
|
|
146
|
+
"clusters",
|
|
147
|
+
"output_records",
|
|
148
|
+
"source1_members",
|
|
149
|
+
"source2_members",
|
|
150
|
+
"output_fasta",
|
|
151
|
+
]
|
|
152
|
+
)
|
|
153
|
+
for summary in summaries:
|
|
154
|
+
writer.writerow(
|
|
155
|
+
[
|
|
156
|
+
summary.label,
|
|
157
|
+
summary.description,
|
|
158
|
+
summary.clusters,
|
|
159
|
+
summary.output_records,
|
|
160
|
+
summary.source1_members,
|
|
161
|
+
summary.source2_members,
|
|
162
|
+
summary.output_path.name,
|
|
163
|
+
]
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def write_log(path: Path, lines: Iterable[str]) -> None:
|
|
168
|
+
with path.open("w", encoding="utf-8") as handle:
|
|
169
|
+
for line in lines:
|
|
170
|
+
handle.write(f"{line}\n")
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@click.command(context_settings=dict(help_option_names=["-h", "--help"]))
|
|
174
|
+
@click.version_option(__version__)
|
|
175
|
+
@click.option("-1", "--fasta-one", "fasta_one", required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path), help="First FASTA file.")
|
|
176
|
+
@click.option("-2", "--fasta-two", "fasta_two", required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path), help="Second FASTA file.")
|
|
177
|
+
@click.option("-o", "--outdir", required=True, type=click.Path(file_okay=False, path_type=Path), help="Output directory.")
|
|
178
|
+
@click.option("-c", "--perc-identity", default=0.99, show_default=True, type=click.FloatRange(min=0.0, max=1.0, min_open=True), help="CD-HIT-EST sequence identity threshold.")
|
|
179
|
+
@click.option("--coverage", default=0.90, show_default=True, type=click.FloatRange(min=0.0, max=1.0, min_open=True), help="Coverage threshold applied to both -aS and -aL.")
|
|
180
|
+
@click.option("-d", "--description-length", default=0, show_default=True, type=int, help="Value passed to cd-hit-est -d.")
|
|
181
|
+
@click.option("--temp-dir", type=click.Path(exists=True, file_okay=False, path_type=Path), help="Override the temporary directory base.")
|
|
182
|
+
@click.option("--keep-temp-dir", is_flag=True, default=False, help="Keep the temporary working directory.")
|
|
183
|
+
@click.option("--all-members", is_flag=True, default=False, help="Write all members of matching clusters instead of only the representative sequence.")
|
|
184
|
+
def compare_otus(
|
|
185
|
+
fasta_one: Path,
|
|
186
|
+
fasta_two: Path,
|
|
187
|
+
outdir: Path,
|
|
188
|
+
perc_identity: float,
|
|
189
|
+
coverage: float,
|
|
190
|
+
description_length: int,
|
|
191
|
+
temp_dir: Path | None,
|
|
192
|
+
keep_temp_dir: bool,
|
|
193
|
+
all_members: bool,
|
|
194
|
+
) -> None:
|
|
195
|
+
"""
|
|
196
|
+
Compare two OTU FASTA files with cd-hit-est.
|
|
197
|
+
"""
|
|
198
|
+
if not has_cdhit_est():
|
|
199
|
+
raise click.ClickException(
|
|
200
|
+
"cd-hit-est is not available in PATH. Please install CD-HIT and try again."
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
outdir.mkdir(parents=True, exist_ok=True)
|
|
204
|
+
prefix1 = input_prefix(fasta_one)
|
|
205
|
+
prefix2 = input_prefix(fasta_two)
|
|
206
|
+
if prefix1 == prefix2:
|
|
207
|
+
raise click.ClickException(
|
|
208
|
+
"Input basenames resolve to the same prefix. Rename the inputs or adjust their filenames."
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
temp_base = temp_dir or Path(os.environ.get("TMP", "/tmp"))
|
|
212
|
+
workdir = Path(tempfile.mkdtemp(prefix="cdhit_compare_otus_", dir=temp_base))
|
|
213
|
+
concatenated_fasta = workdir / "concatenated.fasta"
|
|
214
|
+
output_prefix = workdir / "cdhit_compare_otus"
|
|
215
|
+
log_lines = [
|
|
216
|
+
"cdhit-compare-otus run",
|
|
217
|
+
f"fasta_one\t{fasta_one}",
|
|
218
|
+
f"fasta_two\t{fasta_two}",
|
|
219
|
+
f"outdir\t{outdir}",
|
|
220
|
+
f"temp_dir\t{workdir}",
|
|
221
|
+
f"prefix_one\t{prefix1}",
|
|
222
|
+
f"prefix_two\t{prefix2}",
|
|
223
|
+
f"perc_identity\t{perc_identity}",
|
|
224
|
+
f"coverage\t{coverage}",
|
|
225
|
+
f"description_length\t{description_length}",
|
|
226
|
+
f"all_members\t{all_members}",
|
|
227
|
+
]
|
|
228
|
+
|
|
229
|
+
stats_path = outdir / "compare_otus_stats.tsv"
|
|
230
|
+
log_path = outdir / "compare_otus.log"
|
|
231
|
+
|
|
232
|
+
summaries = {
|
|
233
|
+
"one_to_one_matches": CategorySummary(
|
|
234
|
+
label="one_to_one_matches",
|
|
235
|
+
description="Exactly one member from each input file.",
|
|
236
|
+
output_path=outdir / "one_to_one_matches.fasta",
|
|
237
|
+
),
|
|
238
|
+
"shared_multi": CategorySummary(
|
|
239
|
+
label="shared_multi",
|
|
240
|
+
description="At least one member from both inputs and multiple members in at least one input.",
|
|
241
|
+
output_path=outdir / "shared_multi.fasta",
|
|
242
|
+
),
|
|
243
|
+
"only_file1": CategorySummary(
|
|
244
|
+
label="only_file1",
|
|
245
|
+
description=f"Clusters containing only sequences from {prefix1}.",
|
|
246
|
+
output_path=outdir / f"only_{prefix1}.fasta",
|
|
247
|
+
),
|
|
248
|
+
"only_file2": CategorySummary(
|
|
249
|
+
label="only_file2",
|
|
250
|
+
description=f"Clusters containing only sequences from {prefix2}.",
|
|
251
|
+
output_path=outdir / f"only_{prefix2}.fasta",
|
|
252
|
+
),
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
output_sequences = {key: [] for key in summaries}
|
|
256
|
+
|
|
257
|
+
try:
|
|
258
|
+
renamed_sequences: Dict[str, RenamedSequence] = {}
|
|
259
|
+
renamed_sequences.update(
|
|
260
|
+
write_relabeled_fasta(fasta_one, prefix1, concatenated_fasta, append=False)
|
|
261
|
+
)
|
|
262
|
+
second_sequences = write_relabeled_fasta(
|
|
263
|
+
fasta_two, prefix2, concatenated_fasta, append=True
|
|
264
|
+
)
|
|
265
|
+
overlap = set(renamed_sequences).intersection(second_sequences)
|
|
266
|
+
if overlap:
|
|
267
|
+
duplicated = ", ".join(sorted(overlap))
|
|
268
|
+
raise click.ClickException(f"Duplicate renamed identifiers across inputs: {duplicated}")
|
|
269
|
+
renamed_sequences.update(second_sequences)
|
|
270
|
+
|
|
271
|
+
command = [
|
|
272
|
+
"cd-hit-est",
|
|
273
|
+
"-i",
|
|
274
|
+
str(concatenated_fasta),
|
|
275
|
+
"-o",
|
|
276
|
+
str(output_prefix),
|
|
277
|
+
"-c",
|
|
278
|
+
str(perc_identity),
|
|
279
|
+
"-aS",
|
|
280
|
+
str(coverage),
|
|
281
|
+
"-aL",
|
|
282
|
+
str(coverage),
|
|
283
|
+
"-d",
|
|
284
|
+
str(description_length),
|
|
285
|
+
]
|
|
286
|
+
log_lines.append(f"command\t{' '.join(command)}")
|
|
287
|
+
|
|
288
|
+
result = subprocess.run(command, check=True, capture_output=True, text=True)
|
|
289
|
+
if result.stdout.strip():
|
|
290
|
+
log_lines.append("cd-hit-est stdout:")
|
|
291
|
+
log_lines.extend(result.stdout.strip().splitlines())
|
|
292
|
+
if result.stderr.strip():
|
|
293
|
+
log_lines.append("cd-hit-est stderr:")
|
|
294
|
+
log_lines.extend(result.stderr.strip().splitlines())
|
|
295
|
+
|
|
296
|
+
for cluster in read_cdhit(f"{output_prefix}.clstr"):
|
|
297
|
+
category, count1, count2 = classify_cluster(cluster, prefix1, prefix2)
|
|
298
|
+
sequences = sequences_for_cluster(cluster, renamed_sequences, all_members=all_members)
|
|
299
|
+
output_sequences[category].extend(sequences)
|
|
300
|
+
summaries[category].clusters += 1
|
|
301
|
+
summaries[category].output_records += len(sequences)
|
|
302
|
+
summaries[category].source1_members += count1
|
|
303
|
+
summaries[category].source2_members += count2
|
|
304
|
+
|
|
305
|
+
for summary in summaries.values():
|
|
306
|
+
written = write_fasta_output(summary.output_path, output_sequences[summary.label])
|
|
307
|
+
if written != summary.output_records:
|
|
308
|
+
raise click.ClickException(
|
|
309
|
+
f"Expected to write {summary.output_records} sequences to {summary.output_path.name}, wrote {written}."
|
|
310
|
+
)
|
|
311
|
+
log_lines.append(
|
|
312
|
+
f"output\t{summary.label}\t{summary.output_path}\tclusters={summary.clusters}\tsequences={summary.output_records}"
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
write_stats(stats_path, summaries.values())
|
|
316
|
+
log_lines.append(f"stats\t{stats_path}")
|
|
317
|
+
log_lines.append(f"log\t{log_path}")
|
|
318
|
+
click.echo(f"Results written to {outdir}")
|
|
319
|
+
except subprocess.CalledProcessError as exc:
|
|
320
|
+
if exc.stdout:
|
|
321
|
+
log_lines.append("cd-hit-est stdout:")
|
|
322
|
+
log_lines.extend(exc.stdout.strip().splitlines())
|
|
323
|
+
if exc.stderr:
|
|
324
|
+
log_lines.append("cd-hit-est stderr:")
|
|
325
|
+
log_lines.extend(exc.stderr.strip().splitlines())
|
|
326
|
+
raise click.ClickException("cd-hit-est failed; see compare_otus.log for details.") from exc
|
|
327
|
+
finally:
|
|
328
|
+
write_log(log_path, log_lines)
|
|
329
|
+
if not keep_temp_dir:
|
|
330
|
+
shutil.rmtree(workdir, ignore_errors=True)
|
|
331
|
+
else:
|
|
332
|
+
click.echo(f"Temporary directory kept at {workdir}")
|