pufprobe 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pufprobe/__init__.py +10 -0
- pufprobe/__main__.py +6 -0
- pufprobe/annotate/__init__.py +1 -0
- pufprobe/annotate/backfill.py +247 -0
- pufprobe/annotate/pipeline.py +72 -0
- pufprobe/cli.py +231 -0
- pufprobe/clustering/__init__.py +1 -0
- pufprobe/clustering/mmseqs.py +103 -0
- pufprobe/clustering/pycluster.py +149 -0
- pufprobe/config.py +154 -0
- pufprobe/context/__init__.py +1 -0
- pufprobe/context/microsynteny.py +65 -0
- pufprobe/context/operons.py +74 -0
- pufprobe/context/phyloprofile.py +75 -0
- pufprobe/context/pipeline.py +264 -0
- pufprobe/embed/__init__.py +1 -0
- pufprobe/embed/esm.py +133 -0
- pufprobe/embed/fallback.py +62 -0
- pufprobe/embed/pipeline.py +314 -0
- pufprobe/externals.py +162 -0
- pufprobe/features.py +132 -0
- pufprobe/fold/__init__.py +1 -0
- pufprobe/fold/esmfold.py +147 -0
- pufprobe/fold/pipeline.py +183 -0
- pufprobe/geneqc/__init__.py +1 -0
- pufprobe/geneqc/classifier.py +162 -0
- pufprobe/io_utils.py +82 -0
- pufprobe/logging_utils.py +28 -0
- pufprobe/models.py +239 -0
- pufprobe/ontology.py +66 -0
- pufprobe/prepare/__init__.py +1 -0
- pufprobe/prepare/gbk.py +176 -0
- pufprobe/prepare/gff.py +216 -0
- pufprobe/prepare/pipeline.py +195 -0
- pufprobe/rank/__init__.py +1 -0
- pufprobe/rank/calibrate.py +186 -0
- pufprobe/rank/fuse.py +413 -0
- pufprobe/rank/pipeline.py +139 -0
- pufprobe/rank/priority.py +98 -0
- pufprobe/report/__init__.py +1 -0
- pufprobe/report/html_report.py +270 -0
- pufprobe/report/pipeline.py +113 -0
- pufprobe/search/__init__.py +1 -0
- pufprobe/search/foldseek.py +245 -0
- pufprobe/search/hhblits.py +124 -0
- pufprobe/search/idmapping.py +197 -0
- pufprobe/search/mmseqs_search.py +214 -0
- pufprobe/search/pipeline.py +125 -0
- pufprobe/search/sifts.py +71 -0
- pufprobe/workdir.py +100 -0
- pufprobe-0.5.0.dist-info/METADATA +226 -0
- pufprobe-0.5.0.dist-info/RECORD +56 -0
- pufprobe-0.5.0.dist-info/WHEEL +5 -0
- pufprobe-0.5.0.dist-info/entry_points.txt +2 -0
- pufprobe-0.5.0.dist-info/licenses/LICENSE +21 -0
- pufprobe-0.5.0.dist-info/top_level.txt +1 -0
pufprobe/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""PUFprobe: second-stage functional inference for hypothetical proteins.
|
|
2
|
+
|
|
3
|
+
PUFprobe consumes the output of Bakta, Prokka or NCBI PGAP and adds a
|
|
4
|
+
second-stage inference layer for CDS records annotated as hypothetical
|
|
5
|
+
protein, uncharacterized protein or domains of unknown function.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "0.4.0"
|
|
9
|
+
|
|
10
|
+
__all__ = ["__version__"]
|
pufprobe/__main__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""annotate stage: backfill predictions into GFF3/GBK/FAA files."""
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Backfill high-confidence predictions into new annotation files.
|
|
2
|
+
|
|
3
|
+
Original GFF3 / GenBank / FASTA files are only ever opened for reading. For
|
|
4
|
+
every sample the stage writes parallel files under ``09_annotate``:
|
|
5
|
+
|
|
6
|
+
* ``<sample>.pufprobe.gff3`` — every line copied verbatim except matched CDS
|
|
7
|
+
lines, whose ``product`` attribute is replaced, ``Dbxref`` entries appended
|
|
8
|
+
and a ``PUFprobe=...`` evidence note added;
|
|
9
|
+
* ``<sample>.pufprobe.gbk`` — minimal line-based GenBank rewrite when a
|
|
10
|
+
GenBank source exists (``/product`` replaced, ``/db_xref`` and ``/note``
|
|
11
|
+
appended);
|
|
12
|
+
* ``<sample>.pufprobe.faa`` — protein records renamed via
|
|
13
|
+
:func:`pufprobe.io_utils.write_fasta`.
|
|
14
|
+
|
|
15
|
+
A ``changes.tsv`` ledger records every edit.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import re
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
23
|
+
|
|
24
|
+
from pufprobe.io_utils import read_fasta, write_fasta
|
|
25
|
+
from pufprobe.logging_utils import get_logger
|
|
26
|
+
from pufprobe.models import ProteinPrediction, Sample, TermPrediction
|
|
27
|
+
|
|
28
|
+
log = get_logger("annotate.backfill")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def select_terms(
|
|
32
|
+
predictions: List[ProteinPrediction], min_confidence: float
|
|
33
|
+
) -> Dict[str, Dict[str, Any]]:
|
|
34
|
+
"""Map global key -> accepted edit info for terms >= min_confidence."""
|
|
35
|
+
accepted: Dict[str, Dict[str, Any]] = {}
|
|
36
|
+
for pred in predictions:
|
|
37
|
+
terms = sorted(
|
|
38
|
+
(t for t in pred.terms if t.confidence >= min_confidence),
|
|
39
|
+
key=lambda t: (-t.confidence, t.term_type, t.term_id),
|
|
40
|
+
)
|
|
41
|
+
if not terms:
|
|
42
|
+
continue
|
|
43
|
+
groups = sorted({g for t in terms for g in t.evidence_groups})
|
|
44
|
+
accepted[pred.key] = {
|
|
45
|
+
"new_product": " / ".join(t.term_name for t in terms),
|
|
46
|
+
"confidence": max(t.confidence for t in terms),
|
|
47
|
+
"evidence_groups": groups,
|
|
48
|
+
"term_ids": [f"{t.term_type}:{t.term_id}" for t in terms],
|
|
49
|
+
"terms": terms,
|
|
50
|
+
}
|
|
51
|
+
return accepted
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---------------------------------------------------------------------------
|
|
55
|
+
# GFF3
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
def parse_gff_attributes(attr_string: str) -> Dict[str, str]:
|
|
58
|
+
out: Dict[str, str] = {}
|
|
59
|
+
for part in attr_string.split(";"):
|
|
60
|
+
if not part:
|
|
61
|
+
continue
|
|
62
|
+
key, _, value = part.partition("=")
|
|
63
|
+
out[key] = value
|
|
64
|
+
return out
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def rewrite_gff3(
|
|
68
|
+
src: Path,
|
|
69
|
+
out: Path,
|
|
70
|
+
sample_id: str,
|
|
71
|
+
accepted: Dict[str, Dict[str, Any]],
|
|
72
|
+
) -> List[Dict[str, str]]:
|
|
73
|
+
"""Copy ``src`` to ``out``, editing matched CDS lines. Returns change rows."""
|
|
74
|
+
changes: List[Dict[str, str]] = []
|
|
75
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
76
|
+
with src.open("r", encoding="utf-8", errors="replace") as handle:
|
|
77
|
+
lines = handle.read().splitlines()
|
|
78
|
+
|
|
79
|
+
rewritten: List[str] = []
|
|
80
|
+
for line in lines:
|
|
81
|
+
if not line or line.startswith("#"):
|
|
82
|
+
rewritten.append(line)
|
|
83
|
+
continue
|
|
84
|
+
fields = line.split("\t")
|
|
85
|
+
if len(fields) < 9 or fields[2] != "CDS":
|
|
86
|
+
rewritten.append(line)
|
|
87
|
+
continue
|
|
88
|
+
attrs = parse_gff_attributes(fields[8])
|
|
89
|
+
locus = attrs.get("locus_tag") or attrs.get("ID", "")
|
|
90
|
+
key = f"{sample_id}@{locus}"
|
|
91
|
+
if key not in accepted:
|
|
92
|
+
rewritten.append(line)
|
|
93
|
+
continue
|
|
94
|
+
|
|
95
|
+
edit = accepted[key]
|
|
96
|
+
old_product = attrs.get("product", "")
|
|
97
|
+
attrs["product"] = edit["new_product"]
|
|
98
|
+
|
|
99
|
+
dbxref = [x for x in attrs.get("Dbxref", "").split(",") if x]
|
|
100
|
+
for term_id in edit["term_ids"]:
|
|
101
|
+
if term_id not in dbxref:
|
|
102
|
+
dbxref.append(term_id)
|
|
103
|
+
if dbxref:
|
|
104
|
+
attrs["Dbxref"] = ",".join(dbxref)
|
|
105
|
+
|
|
106
|
+
note = f"PUFprobe:confidence={edit['confidence']:.3f};evidence_groups={'+'.join(edit['evidence_groups'])}"
|
|
107
|
+
if attrs.get("note"):
|
|
108
|
+
attrs["note"] = f"{attrs['note']},{note}"
|
|
109
|
+
else:
|
|
110
|
+
attrs["note"] = note
|
|
111
|
+
|
|
112
|
+
fields[8] = ";".join(f"{k}={v}" for k, v in attrs.items())
|
|
113
|
+
rewritten.append("\t".join(fields))
|
|
114
|
+
changes.append(
|
|
115
|
+
{
|
|
116
|
+
"sample": sample_id,
|
|
117
|
+
"locus": locus,
|
|
118
|
+
"old_product": old_product,
|
|
119
|
+
"new_product": edit["new_product"],
|
|
120
|
+
"confidence": f"{edit['confidence']:.3f}",
|
|
121
|
+
"supporting_terms": ";".join(edit["term_ids"]),
|
|
122
|
+
}
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
with out.open("w", encoding="utf-8", newline="\n") as handle:
|
|
126
|
+
handle.write("\n".join(rewritten) + "\n")
|
|
127
|
+
return changes
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# ---------------------------------------------------------------------------
|
|
131
|
+
# GenBank (minimal line-based rewrite)
|
|
132
|
+
# ---------------------------------------------------------------------------
|
|
133
|
+
_PRODUCT_RE = re.compile(r"^(\s+)/product=")
|
|
134
|
+
_LOCUS_RE = re.compile(r'^\s+/locus_tag="([^"]+)"')
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def rewrite_gbk(
|
|
138
|
+
src: Path,
|
|
139
|
+
out: Path,
|
|
140
|
+
sample_id: str,
|
|
141
|
+
accepted: Dict[str, Dict[str, Any]],
|
|
142
|
+
) -> int:
|
|
143
|
+
"""Minimal GenBank rewrite; returns the number of edited feature blocks."""
|
|
144
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
145
|
+
lines = src.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
146
|
+
rewritten: List[str] = []
|
|
147
|
+
i = 0
|
|
148
|
+
edited = 0
|
|
149
|
+
while i < len(lines):
|
|
150
|
+
line = lines[i]
|
|
151
|
+
if not line.startswith(" ") or line.startswith("FEATURES"):
|
|
152
|
+
rewritten.append(line)
|
|
153
|
+
i += 1
|
|
154
|
+
continue
|
|
155
|
+
# Collect one feature block: a non-qualifier line starts a new feature.
|
|
156
|
+
block = [line]
|
|
157
|
+
j = i + 1
|
|
158
|
+
while j < len(lines):
|
|
159
|
+
nxt = lines[j]
|
|
160
|
+
if nxt.startswith("ORIGIN") or nxt.startswith("//"):
|
|
161
|
+
break
|
|
162
|
+
if not nxt.startswith(" "):
|
|
163
|
+
break
|
|
164
|
+
stripped = nxt.lstrip()
|
|
165
|
+
if not stripped.startswith("/"):
|
|
166
|
+
break
|
|
167
|
+
block.append(nxt)
|
|
168
|
+
j += 1
|
|
169
|
+
|
|
170
|
+
locus = None
|
|
171
|
+
for bl in block:
|
|
172
|
+
m = _LOCUS_RE.match(bl)
|
|
173
|
+
if m:
|
|
174
|
+
locus = m.group(1)
|
|
175
|
+
break
|
|
176
|
+
key = f"{sample_id}@{locus}" if locus else None
|
|
177
|
+
if key is None or key not in accepted:
|
|
178
|
+
rewritten.extend(block)
|
|
179
|
+
else:
|
|
180
|
+
edit = accepted[key]
|
|
181
|
+
for bl in block:
|
|
182
|
+
rewritten.append(bl)
|
|
183
|
+
if _PRODUCT_RE.match(bl):
|
|
184
|
+
indent = _PRODUCT_RE.match(bl).group(1)
|
|
185
|
+
rewritten.append(f'{indent}/db_xref="{";".join(edit["term_ids"])}"')
|
|
186
|
+
rewritten.append(
|
|
187
|
+
f'{indent}/note="PUFprobe:confidence={edit["confidence"]:.3f};'
|
|
188
|
+
f'evidence_groups={"+".join(edit["evidence_groups"])}"'
|
|
189
|
+
)
|
|
190
|
+
edited += 1
|
|
191
|
+
i = j
|
|
192
|
+
out.write_text("\n".join(rewritten) + "\n", encoding="utf-8", newline="\n")
|
|
193
|
+
return edited
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# ---------------------------------------------------------------------------
|
|
197
|
+
# FASTA
|
|
198
|
+
# ---------------------------------------------------------------------------
|
|
199
|
+
def rewrite_faa(
|
|
200
|
+
src: Path,
|
|
201
|
+
out: Path,
|
|
202
|
+
sample_id: str,
|
|
203
|
+
accepted: Dict[str, Dict[str, Any]],
|
|
204
|
+
) -> int:
|
|
205
|
+
"""Rename matched protein records; returns the number of renamed records."""
|
|
206
|
+
renamed = 0
|
|
207
|
+
records = []
|
|
208
|
+
for header, seq in read_fasta(src):
|
|
209
|
+
first = header.split()[0] if header else ""
|
|
210
|
+
new_header = header
|
|
211
|
+
for key, edit in accepted.items():
|
|
212
|
+
_, _, locus = key.partition("@")
|
|
213
|
+
if locus and locus in header:
|
|
214
|
+
new_header = f"{first} {edit['new_product']}"
|
|
215
|
+
renamed += 1
|
|
216
|
+
break
|
|
217
|
+
records.append((new_header, seq))
|
|
218
|
+
write_fasta(out, records)
|
|
219
|
+
return renamed
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def run_backfill(
|
|
223
|
+
out_dir: Path,
|
|
224
|
+
samples: List[Sample],
|
|
225
|
+
accepted: Dict[str, Dict[str, Any]],
|
|
226
|
+
) -> List[Dict[str, str]]:
|
|
227
|
+
"""Backfill every sample; returns the combined change ledger."""
|
|
228
|
+
all_changes: List[Dict[str, str]] = []
|
|
229
|
+
for sample in samples:
|
|
230
|
+
sid = sample.sample_id
|
|
231
|
+
if sample.gff_path:
|
|
232
|
+
changes = rewrite_gff3(
|
|
233
|
+
Path(sample.gff_path), out_dir / f"{sid}.pufprobe.gff3", sid, accepted
|
|
234
|
+
)
|
|
235
|
+
all_changes.extend(changes)
|
|
236
|
+
log.info("%s: %d CDS rows backfilled in GFF3.", sid, len(changes))
|
|
237
|
+
if sample.gbk_path:
|
|
238
|
+
n = rewrite_gbk(
|
|
239
|
+
Path(sample.gbk_path), out_dir / f"{sid}.pufprobe.gbk", sid, accepted
|
|
240
|
+
)
|
|
241
|
+
log.info("%s: %d feature blocks edited in GenBank.", sid, n)
|
|
242
|
+
if sample.faa_path:
|
|
243
|
+
n = rewrite_faa(
|
|
244
|
+
Path(sample.faa_path), out_dir / f"{sid}.pufprobe.faa", sid, accepted
|
|
245
|
+
)
|
|
246
|
+
log.info("%s: %d protein records renamed in FAA.", sid, n)
|
|
247
|
+
return all_changes
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""annotate stage pipeline.
|
|
2
|
+
|
|
3
|
+
Contract implemented here:
|
|
4
|
+
|
|
5
|
+
1. Load predictions and select terms whose confidence is at least
|
|
6
|
+
--min-confidence (default config rank.min_confidence_annotate).
|
|
7
|
+
2. For each sample, produce new annotation files under 09_annotate/:
|
|
8
|
+
- <sample>.pufprobe.gff3: original GFF rows with the product and Dbxref
|
|
9
|
+
fields updated for matched CDS rows, plus a PUFprobe evidence note;
|
|
10
|
+
- <sample>.pufprobe.gbk when a GBK source exists;
|
|
11
|
+
- <sample>.pufprobe.faa: protein records renamed with the new product.
|
|
12
|
+
3. Original files are never modified or overwritten; a mapping table
|
|
13
|
+
09_annotate/changes.tsv records every edit with the supporting term and
|
|
14
|
+
confidence.
|
|
15
|
+
|
|
16
|
+
Completion marker written for annotate.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, Dict, Optional
|
|
21
|
+
|
|
22
|
+
from pufprobe.io_utils import read_json
|
|
23
|
+
from pufprobe.logging_utils import get_logger
|
|
24
|
+
from pufprobe.models import Sample
|
|
25
|
+
from pufprobe.annotate.backfill import run_backfill, select_terms
|
|
26
|
+
from pufprobe.rank.fuse import load_predictions
|
|
27
|
+
from pufprobe.workdir import WorkDir
|
|
28
|
+
|
|
29
|
+
log = get_logger("annotate.pipeline")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def run(
|
|
33
|
+
workdir: WorkDir,
|
|
34
|
+
config: Dict[str, Any],
|
|
35
|
+
min_confidence: Optional[float] = None,
|
|
36
|
+
) -> None:
|
|
37
|
+
threshold = (
|
|
38
|
+
float(min_confidence)
|
|
39
|
+
if min_confidence is not None
|
|
40
|
+
else float(config["rank"]["min_confidence_annotate"])
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
predictions = load_predictions(workdir.path("rank", "predictions.json"))
|
|
44
|
+
samples_path = workdir.path("prepare", "samples.json")
|
|
45
|
+
if not samples_path.exists():
|
|
46
|
+
raise FileNotFoundError(f"missing {samples_path}; run prepare first")
|
|
47
|
+
samples = [Sample(**row) for row in read_json(samples_path)] # type: ignore[arg-type]
|
|
48
|
+
|
|
49
|
+
accepted = select_terms(predictions, threshold)
|
|
50
|
+
log.info(
|
|
51
|
+
"annotate: %d terms pass confidence threshold %.2f across %d proteins.",
|
|
52
|
+
sum(len(edit["terms"]) for edit in accepted.values()),
|
|
53
|
+
threshold,
|
|
54
|
+
len(accepted),
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
out_dir = workdir.stage_dir("annotate")
|
|
58
|
+
changes = run_backfill(out_dir, samples, accepted)
|
|
59
|
+
|
|
60
|
+
changes_path = out_dir / "changes.tsv"
|
|
61
|
+
with changes_path.open("w", encoding="utf-8", newline="\n") as handle:
|
|
62
|
+
handle.write(
|
|
63
|
+
"sample\tlocus\told_product\tnew_product\tconfidence\tsupporting terms\n"
|
|
64
|
+
)
|
|
65
|
+
for row in changes:
|
|
66
|
+
handle.write(
|
|
67
|
+
f"{row['sample']}\t{row['locus']}\t{row['old_product']}\t"
|
|
68
|
+
f"{row['new_product']}\t{row['confidence']}\t{row['supporting_terms']}\n"
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
workdir.mark_complete("annotate")
|
|
72
|
+
log.info("annotate done: %d edits recorded in changes.tsv.", len(changes))
|
pufprobe/cli.py
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"""Command line interface for PUFprobe.
|
|
2
|
+
|
|
3
|
+
Eight pipeline subcommands (prepare, embed, fold, search, context, rank,
|
|
4
|
+
report, annotate) share global options for the working directory, threads, CPU
|
|
5
|
+
fallback, configuration and logging.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, Dict, List, Optional
|
|
10
|
+
|
|
11
|
+
import click
|
|
12
|
+
|
|
13
|
+
from pufprobe import __version__
|
|
14
|
+
from pufprobe.config import load_config
|
|
15
|
+
from pufprobe.workdir import WorkDir
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Context:
|
|
19
|
+
def __init__(self, workdir: WorkDir, config: Dict[str, Any]):
|
|
20
|
+
self.workdir = workdir
|
|
21
|
+
self.config = config
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@click.group(context_settings={"help_option_names": ["-h", "--help"]})
|
|
25
|
+
@click.version_option(__version__, prog_name="pufprobe")
|
|
26
|
+
@click.option(
|
|
27
|
+
"--workdir",
|
|
28
|
+
"-w",
|
|
29
|
+
required=False,
|
|
30
|
+
type=click.Path(file_okay=False, path_type=Path),
|
|
31
|
+
envvar="PUFPROBE_WORKDIR",
|
|
32
|
+
help="Working directory for intermediate and final outputs.",
|
|
33
|
+
)
|
|
34
|
+
@click.option("--threads", "-t", default=1, show_default=True, help="Threads for external tools.")
|
|
35
|
+
@click.option("--cpu", is_flag=True, default=False, help="Force CPU fallback models.")
|
|
36
|
+
@click.option("--config", "config_path", type=click.Path(exists=True, dir_okay=False, path_type=Path), help="JSON config overriding defaults.")
|
|
37
|
+
@click.option("--log-level", default="INFO", show_default=True, help="Logging level.")
|
|
38
|
+
@click.pass_context
|
|
39
|
+
def cli(
|
|
40
|
+
ctx: click.Context,
|
|
41
|
+
workdir: Path,
|
|
42
|
+
threads: int,
|
|
43
|
+
cpu: bool,
|
|
44
|
+
config_path: Optional[Path],
|
|
45
|
+
log_level: str,
|
|
46
|
+
) -> None:
|
|
47
|
+
"""Second-stage functional inference for hypothetical proteins."""
|
|
48
|
+
import sys
|
|
49
|
+
|
|
50
|
+
help_requested = any(arg in ("-h", "--help") for arg in sys.argv[1:])
|
|
51
|
+
if workdir is None:
|
|
52
|
+
if help_requested:
|
|
53
|
+
ctx.obj = None
|
|
54
|
+
return
|
|
55
|
+
if ctx.invoked_subcommand == "doctor":
|
|
56
|
+
ctx.obj = Context(None, load_config(config_path))
|
|
57
|
+
return
|
|
58
|
+
raise click.UsageError("Missing option '--workdir' / '-w'.")
|
|
59
|
+
config = load_config(config_path)
|
|
60
|
+
wd = WorkDir(workdir, cpu=cpu, threads=threads, log_level=log_level)
|
|
61
|
+
wd.write_metadata()
|
|
62
|
+
ctx.obj = Context(wd, config)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _resume_guard(wd: WorkDir, stage: str, force: bool) -> bool:
|
|
66
|
+
if wd.is_complete(stage) and not force:
|
|
67
|
+
click.echo(f"[skip] {stage} already complete (use --force to rerun).")
|
|
68
|
+
return True
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@cli.command()
|
|
73
|
+
@click.option("--input", "inputs", multiple=True, type=click.Path(exists=True, path_type=Path), help="Input GFF3/GBK/FAA file or directory; repeatable.")
|
|
74
|
+
@click.option("--input-list", type=click.Path(exists=True, dir_okay=False, path_type=Path), help="Text file with one input path per line.")
|
|
75
|
+
@click.option("--include-smorf", is_flag=True, default=False, help="Keep sub-30 aa candidates via the permissive smORF path.")
|
|
76
|
+
@click.option("--force", is_flag=True, default=False, help="Ignore the completion marker.")
|
|
77
|
+
@click.pass_obj
|
|
78
|
+
def prepare(obj: Context, inputs: tuple, input_list: Optional[Path], include_smorf: bool, force: bool) -> None:
|
|
79
|
+
"""Parse Bakta/Prokka outputs and run true-gene quality control."""
|
|
80
|
+
from pufprobe.prepare.pipeline import run
|
|
81
|
+
|
|
82
|
+
if _resume_guard(obj.workdir, "prepare", force):
|
|
83
|
+
return
|
|
84
|
+
paths = _collect_inputs(list(inputs), input_list)
|
|
85
|
+
if not paths:
|
|
86
|
+
raise click.UsageError("Provide at least one --input path or --input-list.")
|
|
87
|
+
run(obj.workdir, obj.config, paths, include_smorf=include_smorf)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@cli.command()
|
|
91
|
+
@click.option("--model", default=None, help="ESM-2 checkpoint name (defaults from config).")
|
|
92
|
+
@click.option("--precomputed", type=click.Path(exists=True, path_type=Path), help="Directory or .npz with precomputed embeddings.")
|
|
93
|
+
@click.option("--fallback-only", is_flag=True, default=False, help="Use sequence feature fallback without torch.")
|
|
94
|
+
@click.option("--force", is_flag=True, default=False)
|
|
95
|
+
@click.pass_obj
|
|
96
|
+
def embed(obj: Context, model: Optional[str], precomputed: Optional[Path], fallback_only: bool, force: bool) -> None:
|
|
97
|
+
"""Compute embeddings, cluster families and select representatives."""
|
|
98
|
+
from pufprobe.embed.pipeline import run
|
|
99
|
+
|
|
100
|
+
if _resume_guard(obj.workdir, "embed", force):
|
|
101
|
+
return
|
|
102
|
+
run(obj.workdir, obj.config, model=model, precomputed=precomputed, fallback_only=fallback_only)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@cli.command()
|
|
106
|
+
@click.option("--engine", type=click.Choice(["esmfold", "af2"]), default=None, help="Structure prediction engine.")
|
|
107
|
+
@click.option("--precomputed", type=click.Path(exists=True, file_okay=False, path_type=Path), help="Directory of PDB files named by family id.")
|
|
108
|
+
@click.option("--force", is_flag=True, default=False)
|
|
109
|
+
@click.pass_obj
|
|
110
|
+
def fold(obj: Context, engine: Optional[str], precomputed: Optional[Path], force: bool) -> None:
|
|
111
|
+
"""Fold family representatives and apply the pLDDT quality gate."""
|
|
112
|
+
from pufprobe.fold.pipeline import run
|
|
113
|
+
|
|
114
|
+
if _resume_guard(obj.workdir, "fold", force):
|
|
115
|
+
return
|
|
116
|
+
run(obj.workdir, obj.config, engine=engine, precomputed=precomputed)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@cli.command()
|
|
120
|
+
@click.option("--channels", default=None, help="Comma-separated channels: mmseqs,foldseek,hhblits,tmvec.")
|
|
121
|
+
@click.option("--dbdir", type=click.Path(exists=True, file_okay=False, path_type=Path), help="Directory holding versioned search databases.")
|
|
122
|
+
@click.option("--force", is_flag=True, default=False)
|
|
123
|
+
@click.pass_obj
|
|
124
|
+
def search(obj: Context, channels: Optional[str], dbdir: Optional[Path], force: bool) -> None:
|
|
125
|
+
"""Run structure/sequence remote-homology searches into unified evidence."""
|
|
126
|
+
from pufprobe.search.pipeline import run
|
|
127
|
+
|
|
128
|
+
if _resume_guard(obj.workdir, "search", force):
|
|
129
|
+
return
|
|
130
|
+
channel_list = [c.strip() for c in channels.split(",")] if channels else None
|
|
131
|
+
run(obj.workdir, obj.config, channels=channel_list, dbdir=dbdir)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@cli.command()
|
|
135
|
+
@click.option("--card", type=click.Path(exists=True, path_type=Path), help="CARD reference (FASTA or ARO table) for neighbourhood checks.")
|
|
136
|
+
@click.option("--vfdb", type=click.Path(exists=True, path_type=Path), help="VFDB reference for neighbourhood checks.")
|
|
137
|
+
@click.option("--reference-fna", "reference_fna", type=click.Path(exists=True, file_okay=False, path_type=Path), help="Directory of genome FASTA files for micro-synteny.")
|
|
138
|
+
@click.option("--force", is_flag=True, default=False)
|
|
139
|
+
@click.pass_obj
|
|
140
|
+
def context(obj: Context, card: Optional[Path], vfdb: Optional[Path], reference_fna: Optional[Path], force: bool) -> None:
|
|
141
|
+
"""Compute operon, phylogenetic-profile and micro-synteny evidence."""
|
|
142
|
+
from pufprobe.context.pipeline import run
|
|
143
|
+
|
|
144
|
+
if _resume_guard(obj.workdir, "context", force):
|
|
145
|
+
return
|
|
146
|
+
run(obj.workdir, obj.config, card=card, vfdb=vfdb, reference_fna=reference_fna)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@cli.command()
|
|
150
|
+
@click.option("--weights", type=click.Path(exists=True, path_type=Path), help="Trained fusion model weights; absent uses rule weights.")
|
|
151
|
+
@click.option("--force", is_flag=True, default=False)
|
|
152
|
+
@click.pass_obj
|
|
153
|
+
def rank(obj: Context, weights: Optional[Path], force: bool) -> None:
|
|
154
|
+
"""Fuse evidence, calibrate confidence and rank wet-lab priorities."""
|
|
155
|
+
from pufprobe.rank.pipeline import run
|
|
156
|
+
|
|
157
|
+
if _resume_guard(obj.workdir, "rank", force):
|
|
158
|
+
return
|
|
159
|
+
run(obj.workdir, obj.config, weights=weights)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@cli.command()
|
|
163
|
+
@click.option("--formats", default="json,html,md", show_default=True, help="Comma-separated output formats.")
|
|
164
|
+
@click.option("--force", is_flag=True, default=False)
|
|
165
|
+
@click.pass_obj
|
|
166
|
+
def report(obj: Context, formats: str, force: bool) -> None:
|
|
167
|
+
"""Write JSON and auditable HTML/Markdown evidence reports."""
|
|
168
|
+
from pufprobe.report.pipeline import run
|
|
169
|
+
|
|
170
|
+
if _resume_guard(obj.workdir, "report", force):
|
|
171
|
+
return
|
|
172
|
+
run(obj.workdir, obj.config, formats=[f.strip() for f in formats.split(",")])
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@cli.command()
|
|
176
|
+
@click.option("--min-confidence", default=None, type=float, help="Override the annotation confidence threshold.")
|
|
177
|
+
@click.option("--force", is_flag=True, default=False)
|
|
178
|
+
@click.pass_obj
|
|
179
|
+
def annotate(obj: Context, min_confidence: Optional[float], force: bool) -> None:
|
|
180
|
+
"""Backfill high-confidence predictions into new GFF3/GBK/FAA files."""
|
|
181
|
+
from pufprobe.annotate.pipeline import run
|
|
182
|
+
|
|
183
|
+
if _resume_guard(obj.workdir, "annotate", force):
|
|
184
|
+
return
|
|
185
|
+
run(obj.workdir, obj.config, min_confidence=min_confidence)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@cli.command()
|
|
189
|
+
@click.pass_obj
|
|
190
|
+
def doctor(obj: Context) -> None:
|
|
191
|
+
"""Report detected external tools, Python modules and GPU availability."""
|
|
192
|
+
from pufprobe.externals import summary
|
|
193
|
+
import json
|
|
194
|
+
|
|
195
|
+
click.echo(json.dumps(summary(), indent=2, ensure_ascii=False))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _collect_inputs(inputs: List[Path], input_list: Optional[Path]) -> List[Path]:
|
|
199
|
+
paths = list(inputs)
|
|
200
|
+
if input_list is not None:
|
|
201
|
+
for line in input_list.read_text(encoding="utf-8").splitlines():
|
|
202
|
+
line = line.strip()
|
|
203
|
+
if line and not line.startswith("#"):
|
|
204
|
+
paths.append(Path(line))
|
|
205
|
+
expanded: List[Path] = []
|
|
206
|
+
for path in paths:
|
|
207
|
+
if not path.exists():
|
|
208
|
+
raise click.UsageError(f"Input path does not exist: {path}")
|
|
209
|
+
if path.is_dir():
|
|
210
|
+
for pattern in ("*.gff", "*.gff3"):
|
|
211
|
+
expanded.extend(sorted(path.rglob(pattern)))
|
|
212
|
+
for pattern in ("*.gbk", "*.gbff", "*.genbank"):
|
|
213
|
+
expanded.extend(sorted(path.rglob(pattern)))
|
|
214
|
+
for faa in sorted(path.rglob("*.faa")):
|
|
215
|
+
# Bakta also writes <prefix>.hypotheticals.faa (plural), a
|
|
216
|
+
# subset holding only hypothetical proteins. Indexing that
|
|
217
|
+
# subset would make every annotated protein appear unaligned,
|
|
218
|
+
# so skip it and keep the complete .faa. Both the released
|
|
219
|
+
# plural and the singular spelling are tolerated.
|
|
220
|
+
if faa.name.endswith(".hypotheticals.faa") or faa.name.endswith(
|
|
221
|
+
".hypothetical.faa"
|
|
222
|
+
):
|
|
223
|
+
continue
|
|
224
|
+
expanded.append(faa)
|
|
225
|
+
else:
|
|
226
|
+
expanded.append(path)
|
|
227
|
+
return sorted(set(expanded))
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
if __name__ == "__main__":
|
|
231
|
+
cli()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""clustering backends: MMseqs2 and pure-Python graph fallback."""
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""MMseqs2 sequence clustering (``createdb`` / ``cluster`` / ``createtsv``).
|
|
2
|
+
|
|
3
|
+
When the ``mmseqs`` binary is detected on PATH this runs greedy cascaded
|
|
4
|
+
clustering in a throw-away temp directory and returns a unified
|
|
5
|
+
``{global_key: cluster_index}`` mapping. When the binary is missing the
|
|
6
|
+
function returns ``None`` (after logging install guidance) so the caller can
|
|
7
|
+
drop to the pure-numpy :mod:`pufprobe.clustering.pycluster` path.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import subprocess
|
|
13
|
+
import tempfile
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Dict, Optional
|
|
16
|
+
|
|
17
|
+
from pufprobe import externals
|
|
18
|
+
from pufprobe.logging_utils import get_logger
|
|
19
|
+
|
|
20
|
+
logger = get_logger(__name__)
|
|
21
|
+
|
|
22
|
+
INSTALL_HINT = "Install MMseqs2 with `conda install -c conda-forge mmseqs2` (see https://mmseqs.com)."
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def run_mmseqs_cluster(
|
|
26
|
+
fasta_path: Path,
|
|
27
|
+
work_dir: Path,
|
|
28
|
+
identity: float,
|
|
29
|
+
coverage: float,
|
|
30
|
+
threads: int = 1,
|
|
31
|
+
) -> Optional[Dict[str, int]]:
|
|
32
|
+
"""Cluster sequences in ``fasta_path``; return ``{key: cluster_index}``.
|
|
33
|
+
|
|
34
|
+
Returns ``None`` when the mmseqs binary is unavailable so the caller can
|
|
35
|
+
fall back to pycluster. FASTA headers must be exactly the global keys.
|
|
36
|
+
"""
|
|
37
|
+
info = externals.mmseqs()
|
|
38
|
+
if not info.available:
|
|
39
|
+
logger.warning(
|
|
40
|
+
"mmseqs binary not found; skipping MMseqs2 clustering and falling "
|
|
41
|
+
"back to pure-numpy clustering. %s", INSTALL_HINT,
|
|
42
|
+
)
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
exe = info.path or "mmseqs"
|
|
46
|
+
work_dir.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
tmp = Path(tempfile.mkdtemp(prefix="mmseqs_cluster_", dir=str(work_dir)))
|
|
48
|
+
qdb = tmp / "qdb"
|
|
49
|
+
cdb = tmp / "clu"
|
|
50
|
+
tsv = tmp / "clu.tsv"
|
|
51
|
+
|
|
52
|
+
cmd_base = [exe]
|
|
53
|
+
try:
|
|
54
|
+
_run(cmd_base + ["createdb", str(fasta_path), str(qdb)])
|
|
55
|
+
_run(
|
|
56
|
+
cmd_base
|
|
57
|
+
+ [
|
|
58
|
+
"cluster",
|
|
59
|
+
str(qdb),
|
|
60
|
+
str(cdb),
|
|
61
|
+
str(tmp / "mm_tmp"),
|
|
62
|
+
"--min-seq-id",
|
|
63
|
+
str(identity),
|
|
64
|
+
"-c",
|
|
65
|
+
str(coverage),
|
|
66
|
+
"--threads",
|
|
67
|
+
str(threads),
|
|
68
|
+
]
|
|
69
|
+
)
|
|
70
|
+
_run(cmd_base + ["createtsv", str(qdb), str(qdb), str(cdb), str(tsv)])
|
|
71
|
+
except (subprocess.SubprocessError, OSError) as exc:
|
|
72
|
+
logger.warning("MMseqs2 clustering failed (%s); falling back to pycluster.", exc)
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
return parse_clusters_tsv(tsv)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def parse_clusters_tsv(tsv_path: Path) -> Dict[str, int]:
|
|
79
|
+
"""Parse ``createtsv`` output (rep, member per line) into key->index."""
|
|
80
|
+
mapping: Dict[str, int] = {}
|
|
81
|
+
cluster_order: Dict[str, int] = {}
|
|
82
|
+
next_index = 0
|
|
83
|
+
if not tsv_path.exists():
|
|
84
|
+
return mapping
|
|
85
|
+
with tsv_path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
86
|
+
for line in handle:
|
|
87
|
+
parts = line.rstrip("\n").split("\t")
|
|
88
|
+
if len(parts) < 2:
|
|
89
|
+
continue
|
|
90
|
+
rep, member = parts[0], parts[1]
|
|
91
|
+
if rep not in cluster_order:
|
|
92
|
+
cluster_order[rep] = next_index
|
|
93
|
+
next_index += 1
|
|
94
|
+
mapping[member] = cluster_order[rep]
|
|
95
|
+
return mapping
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _run(cmd: list) -> None:
|
|
99
|
+
completed = subprocess.run(cmd, capture_output=True, text=True, check=True)
|
|
100
|
+
if completed.returncode != 0:
|
|
101
|
+
raise subprocess.CalledProcessError(
|
|
102
|
+
completed.returncode, cmd, completed.stdout, completed.stderr
|
|
103
|
+
)
|