gaphack 0.7.1__tar.gz → 0.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gaphack-0.7.1 → gaphack-0.7.2}/PKG-INFO +8 -1
- {gaphack-0.7.1 → gaphack-0.7.2}/README.md +7 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/__init__.py +1 -1
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/blast_analysis.py +318 -16
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/blast_cli.py +26 -1
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/distance_providers.py +23 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/target_clustering.py +14 -2
- gaphack-0.7.2/gaphack/version.py +7 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack.egg-info/PKG-INFO +8 -1
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack.egg-info/SOURCES.txt +1 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/pyproject.toml +1 -1
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_blast_analysis.py +159 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_distance_providers.py +17 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_target_clustering.py +102 -1
- gaphack-0.7.2/tests/test_version.py +13 -0
- gaphack-0.7.1/gaphack/version.py +0 -6
- {gaphack-0.7.1 → gaphack-0.7.2}/LICENSE +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/analyze.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/analyze_cli.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/blast_neighborhood.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/cli.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/cluster_graph.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/cluster_refinement.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/core.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/neighborhood_finder.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/refine_cli.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/refinement_types.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/utils.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack/vsearch_neighborhood.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack.egg-info/dependency_links.txt +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack.egg-info/entry_points.txt +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack.egg-info/requires.txt +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/gaphack.egg-info/top_level.txt +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/setup.cfg +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_analyze.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_blast_neighborhood.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_cli.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_cluster_graph.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_cluster_refinement.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_core.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_iterative_russula.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_msa_distance_provider.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_pair_scoring.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_refine_cli.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_refine_integration.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_two_pass_refinement.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_utils.py +0 -0
- {gaphack-0.7.1 → gaphack-0.7.2}/tests/test_vsearch.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gaphack
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.2
|
|
4
4
|
Summary: Gap-Optimized Hierarchical Agglomerative Clustering for DNA barcoding
|
|
5
5
|
Author-email: Josh Walker <joshowalker@yahoo.com>
|
|
6
6
|
License: BSD-2-Clause
|
|
@@ -280,6 +280,8 @@ gaphack input.fasta --target seeds.fasta -o target_results
|
|
|
280
280
|
|
|
281
281
|
Target mode focuses on growing one cluster from the provided seed sequences, making it suitable for cases where you want to extract sequences similar to specific targets without attempting to cluster all remaining sequences.
|
|
282
282
|
|
|
283
|
+
The cluster grows one sequence at a time, closest first, up to `--max-lump`. The result is the size with the widest gap among those that include every merge made within `--min-split`: sequences that close to the cluster always join it.
|
|
284
|
+
|
|
283
285
|
### Iterative Refinement (`gaphack-refine`)
|
|
284
286
|
|
|
285
287
|
The `gaphack-refine` tool optimizes cluster boundaries through iterative neighborhood-based refinement. It works with any input clustering (vsearch, CD-HIT, MMseqs2, or gaphack output).
|
|
@@ -420,6 +422,9 @@ gaphack-blast sequences.fasta --format text
|
|
|
420
422
|
|
|
421
423
|
# Tab-separated for spreadsheets
|
|
422
424
|
gaphack-blast sequences.fasta --format tsv
|
|
425
|
+
|
|
426
|
+
# Also find the cluster nearest to the query's and compare the two
|
|
427
|
+
gaphack-blast sequences.fasta --nearest-cluster
|
|
423
428
|
```
|
|
424
429
|
|
|
425
430
|
**Key output fields:**
|
|
@@ -427,6 +432,8 @@ gaphack-blast sequences.fasta --format tsv
|
|
|
427
432
|
- `identity_to_query`: MycoBLAST-adjusted identity percentage
|
|
428
433
|
- `barcode_gap_found`: Whether a clear species boundary was detected
|
|
429
434
|
- `gap_size_percent`: Magnitude of the barcode gap
|
|
435
|
+
- `nearest_cluster` (with `--nearest-cluster`): the cluster grown from the closest
|
|
436
|
+
sequence outside the query cluster, and the barcode gap between the two at P90, P95 and P100
|
|
430
437
|
|
|
431
438
|
For detailed field documentation and integration patterns, see [docs/blast_analysis_integration.md](docs/blast_analysis_integration.md).
|
|
432
439
|
|
|
@@ -243,6 +243,8 @@ gaphack input.fasta --target seeds.fasta -o target_results
|
|
|
243
243
|
|
|
244
244
|
Target mode focuses on growing one cluster from the provided seed sequences, making it suitable for cases where you want to extract sequences similar to specific targets without attempting to cluster all remaining sequences.
|
|
245
245
|
|
|
246
|
+
The cluster grows one sequence at a time, closest first, up to `--max-lump`. The result is the size with the widest gap among those that include every merge made within `--min-split`: sequences that close to the cluster always join it.
|
|
247
|
+
|
|
246
248
|
### Iterative Refinement (`gaphack-refine`)
|
|
247
249
|
|
|
248
250
|
The `gaphack-refine` tool optimizes cluster boundaries through iterative neighborhood-based refinement. It works with any input clustering (vsearch, CD-HIT, MMseqs2, or gaphack output).
|
|
@@ -383,6 +385,9 @@ gaphack-blast sequences.fasta --format text
|
|
|
383
385
|
|
|
384
386
|
# Tab-separated for spreadsheets
|
|
385
387
|
gaphack-blast sequences.fasta --format tsv
|
|
388
|
+
|
|
389
|
+
# Also find the cluster nearest to the query's and compare the two
|
|
390
|
+
gaphack-blast sequences.fasta --nearest-cluster
|
|
386
391
|
```
|
|
387
392
|
|
|
388
393
|
**Key output fields:**
|
|
@@ -390,6 +395,8 @@ gaphack-blast sequences.fasta --format tsv
|
|
|
390
395
|
- `identity_to_query`: MycoBLAST-adjusted identity percentage
|
|
391
396
|
- `barcode_gap_found`: Whether a clear species boundary was detected
|
|
392
397
|
- `gap_size_percent`: Magnitude of the barcode gap
|
|
398
|
+
- `nearest_cluster` (with `--nearest-cluster`): the cluster grown from the closest
|
|
399
|
+
sequence outside the query cluster, and the barcode gap between the two at P90, P95 and P100
|
|
393
400
|
|
|
394
401
|
For detailed field documentation and integration patterns, see [docs/blast_analysis_integration.md](docs/blast_analysis_integration.md).
|
|
395
402
|
|
|
@@ -5,7 +5,7 @@ A Python package for DNA barcode clustering that optimizes for the barcode gap
|
|
|
5
5
|
between intra-species and inter-species genetic distances.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
from .version import __version__
|
|
9
9
|
|
|
10
10
|
from .core import GapOptimizedClustering
|
|
11
11
|
from .target_clustering import TargetModeClustering
|
|
@@ -7,11 +7,11 @@ the query using gap-optimized clustering.
|
|
|
7
7
|
|
|
8
8
|
import logging
|
|
9
9
|
from dataclasses import dataclass, field
|
|
10
|
-
from typing import List, Dict, Optional, Any
|
|
10
|
+
from typing import List, Dict, Optional, Any, Tuple
|
|
11
11
|
import numpy as np
|
|
12
12
|
|
|
13
13
|
from .target_clustering import TargetModeClustering
|
|
14
|
-
from .distance_providers import MSACachedDistanceProvider
|
|
14
|
+
from .distance_providers import MSACachedDistanceProvider, SubsetDistanceProvider
|
|
15
15
|
from .utils import MSAAlignmentError
|
|
16
16
|
|
|
17
17
|
logger = logging.getLogger(__name__)
|
|
@@ -35,6 +35,120 @@ class HistogramData:
|
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
|
|
38
|
+
# Percentiles at which the query cluster is compared with the nearest cluster
|
|
39
|
+
GAP_PERCENTILES = (90, 95, 100)
|
|
40
|
+
|
|
41
|
+
# Named percentiles reported for every identity distribution
|
|
42
|
+
_SUMMARY_PERCENTILES = (("p5", 5), ("p10", 10), ("p25", 25), ("median", 50),
|
|
43
|
+
("p75", 75), ("p90", 90), ("p95", 95))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _round_identity(val: Optional[float]) -> Optional[float]:
|
|
47
|
+
"""Round identity to 2 decimal places (e.g., 99.63%)."""
|
|
48
|
+
return round(val, 2) if val is not None else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class IdentityDistribution:
|
|
53
|
+
"""Identity % (normalized, as used by clustering) of every pair in a set of pairs."""
|
|
54
|
+
values: List[float]
|
|
55
|
+
|
|
56
|
+
def summary(self) -> Dict[str, Optional[float]]:
|
|
57
|
+
"""count, min, p5, p10, p25, median, p75, p90, p95, max and mean (None when empty)."""
|
|
58
|
+
result: Dict[str, Optional[float]] = {"count": len(self.values)}
|
|
59
|
+
if not self.values:
|
|
60
|
+
result.update({"min": None, **{name: None for name, _ in _SUMMARY_PERCENTILES},
|
|
61
|
+
"max": None, "mean": None})
|
|
62
|
+
return result
|
|
63
|
+
arr = np.asarray(self.values, dtype=float)
|
|
64
|
+
result["min"] = float(arr.min())
|
|
65
|
+
for name, pct in _SUMMARY_PERCENTILES:
|
|
66
|
+
result[name] = float(np.percentile(arr, pct))
|
|
67
|
+
result["max"] = float(arr.max())
|
|
68
|
+
result["mean"] = float(arr.mean())
|
|
69
|
+
return result
|
|
70
|
+
|
|
71
|
+
def summary_dict(self) -> Dict[str, Any]:
|
|
72
|
+
"""summary() rounded for JSON."""
|
|
73
|
+
return {k: (v if k == "count" else _round_identity(v)) for k, v in self.summary().items()}
|
|
74
|
+
|
|
75
|
+
def histogram(self, bin_width: float = 0.5) -> Optional["HistogramData"]:
|
|
76
|
+
return _build_histogram(self.values, bin_width=bin_width)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class PercentileGap:
|
|
81
|
+
"""Barcode gap at one percentile, in identity percentage points.
|
|
82
|
+
|
|
83
|
+
At percentile P, the query cluster's within-cluster identities are cut at their
|
|
84
|
+
(100-P)th percentile and the between-cluster identities at their Pth percentile, so
|
|
85
|
+
P100 compares the lowest identity within the query cluster with the highest identity
|
|
86
|
+
between the clusters. A positive gap means the clusters separate at that percentile.
|
|
87
|
+
"""
|
|
88
|
+
percentile: int
|
|
89
|
+
intra_lower_identity: float
|
|
90
|
+
inter_upper_identity: float
|
|
91
|
+
gap_size_percent: float
|
|
92
|
+
gap_exists: bool
|
|
93
|
+
|
|
94
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
95
|
+
return {
|
|
96
|
+
"intra_lower_identity": _round_identity(self.intra_lower_identity),
|
|
97
|
+
"inter_upper_identity": _round_identity(self.inter_upper_identity),
|
|
98
|
+
"gap_size_percent": _round_identity(self.gap_size_percent),
|
|
99
|
+
"gap_exists": self.gap_exists,
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _percentile_gap(intra: List[float], inter: List[float], percentile: int) -> Optional[PercentileGap]:
|
|
104
|
+
"""Gap between within-cluster and between-cluster identities at one percentile."""
|
|
105
|
+
if not intra or not inter:
|
|
106
|
+
return None
|
|
107
|
+
intra_lower = float(np.percentile(intra, 100 - percentile))
|
|
108
|
+
inter_upper = float(np.percentile(inter, percentile))
|
|
109
|
+
gap = intra_lower - inter_upper
|
|
110
|
+
return PercentileGap(percentile=percentile, intra_lower_identity=intra_lower,
|
|
111
|
+
inter_upper_identity=inter_upper, gap_size_percent=gap,
|
|
112
|
+
gap_exists=gap > 0)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@dataclass
|
|
116
|
+
class NearestClusterResult:
|
|
117
|
+
"""The cluster nearest to the query cluster, and how the two compare.
|
|
118
|
+
|
|
119
|
+
The seed is the sequence outside the query cluster that is closest to it (the same
|
|
120
|
+
percentile linkage target mode uses to grow a cluster). The nearest cluster is then
|
|
121
|
+
grown from that seed by target mode over the sequences outside the query cluster,
|
|
122
|
+
with the same thresholds and the same alignment.
|
|
123
|
+
"""
|
|
124
|
+
seed_index: int
|
|
125
|
+
seed_id: str
|
|
126
|
+
seed_linkage_identity: Optional[float] # Identity % of the seed to the query cluster (linkage)
|
|
127
|
+
member_indices: List[int] # 0-based input positions, sorted
|
|
128
|
+
medoid_index: int
|
|
129
|
+
medoid_id: str
|
|
130
|
+
nearest_pairwise: IdentityDistribution # Pairs within the nearest cluster
|
|
131
|
+
query_pairwise: IdentityDistribution # Pairs within the query cluster
|
|
132
|
+
between: IdentityDistribution # Every query-cluster member against every nearest-cluster member
|
|
133
|
+
gaps: Dict[int, Optional[PercentileGap]] # Keyed by percentile (GAP_PERCENTILES)
|
|
134
|
+
|
|
135
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
136
|
+
return {
|
|
137
|
+
"seed_id": self.seed_id,
|
|
138
|
+
"seed_index": self.seed_index,
|
|
139
|
+
"seed_linkage_identity": _round_identity(self.seed_linkage_identity),
|
|
140
|
+
"size": len(self.member_indices),
|
|
141
|
+
"medoid_id": self.medoid_id,
|
|
142
|
+
"medoid_index": self.medoid_index,
|
|
143
|
+
"pairwise_identity": self.nearest_pairwise.summary_dict(),
|
|
144
|
+
"comparison": {
|
|
145
|
+
"query_cluster_pairwise_identity": self.query_pairwise.summary_dict(),
|
|
146
|
+
"between_clusters_identity": self.between.summary_dict(),
|
|
147
|
+
"gaps": {f"p{p}": (gap.to_dict() if gap else None) for p, gap in self.gaps.items()},
|
|
148
|
+
},
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
|
|
38
152
|
@dataclass
|
|
39
153
|
class SequenceResult:
|
|
40
154
|
"""Result for a single sequence in the BLAST analysis."""
|
|
@@ -45,6 +159,7 @@ class SequenceResult:
|
|
|
45
159
|
identity_to_query_normalized: Optional[float] # Normalized identity % (used by clustering)
|
|
46
160
|
identity_to_medoid_normalized: Optional[float] # Identity % to query cluster medoid
|
|
47
161
|
identity_to_nearest_non_member_normalized: Optional[float] # For members: identity % to nearest non-member
|
|
162
|
+
in_nearest_cluster: Optional[bool] = None # None unless a nearest cluster was found
|
|
48
163
|
|
|
49
164
|
|
|
50
165
|
@dataclass
|
|
@@ -85,18 +200,23 @@ class BlastAnalysisResult:
|
|
|
85
200
|
intra_cluster_histogram: Optional[HistogramData] = None
|
|
86
201
|
inter_cluster_histogram: Optional[HistogramData] = None
|
|
87
202
|
|
|
203
|
+
# Linkage/gap percentile used to grow the cluster(s)
|
|
204
|
+
target_percentile: Optional[int] = None
|
|
205
|
+
|
|
206
|
+
# Nearest cluster (only when requested; None if none could be formed)
|
|
207
|
+
nearest_cluster_requested: bool = False
|
|
208
|
+
nearest_cluster: Optional[NearestClusterResult] = None
|
|
209
|
+
|
|
88
210
|
def to_dict(self) -> Dict[str, Any]:
|
|
89
211
|
"""Convert to dictionary for JSON serialization."""
|
|
90
|
-
|
|
91
|
-
"""Round identity to 2 decimal places (e.g., 99.63%)."""
|
|
92
|
-
return round(val, 2) if val is not None else None
|
|
212
|
+
round_identity = _round_identity
|
|
93
213
|
|
|
94
214
|
# Round intra_cluster_identity values
|
|
95
215
|
intra_rounded = {
|
|
96
216
|
k: round_identity(v) for k, v in self.intra_cluster_identity.items()
|
|
97
217
|
}
|
|
98
218
|
|
|
99
|
-
|
|
219
|
+
result = {
|
|
100
220
|
"query": {
|
|
101
221
|
"id": self.query_id,
|
|
102
222
|
"length": self.query_length
|
|
@@ -127,6 +247,7 @@ class BlastAnalysisResult:
|
|
|
127
247
|
"method": self.method,
|
|
128
248
|
"min_split": self.min_split,
|
|
129
249
|
"max_lump": self.max_lump,
|
|
250
|
+
"target_percentile": self.target_percentile,
|
|
130
251
|
"normalization_length": self.normalization_length,
|
|
131
252
|
"identity_metric": self.distance_metric,
|
|
132
253
|
"warnings": self.warnings,
|
|
@@ -137,6 +258,21 @@ class BlastAnalysisResult:
|
|
|
137
258
|
}
|
|
138
259
|
}
|
|
139
260
|
|
|
261
|
+
# Nearest-cluster output appears only when it was requested, so the default
|
|
262
|
+
# output is unchanged; when requested, null marks "no nearest cluster found".
|
|
263
|
+
if self.nearest_cluster_requested:
|
|
264
|
+
nearest = self.nearest_cluster
|
|
265
|
+
result["nearest_cluster"] = nearest.to_dict() if nearest else None
|
|
266
|
+
for seq_dict, seq in zip(result["sequences"], self.sequences):
|
|
267
|
+
seq_dict["in_nearest_cluster"] = seq.in_nearest_cluster
|
|
268
|
+
histograms = result["diagnostics"]["histograms"]
|
|
269
|
+
nearest_hist = nearest.nearest_pairwise.histogram() if nearest else None
|
|
270
|
+
between_hist = nearest.between.histogram() if nearest else None
|
|
271
|
+
histograms["nearest_cluster_intra"] = nearest_hist.to_dict() if nearest_hist else None
|
|
272
|
+
histograms["between_clusters"] = between_hist.to_dict() if between_hist else None
|
|
273
|
+
|
|
274
|
+
return result
|
|
275
|
+
|
|
140
276
|
|
|
141
277
|
def _distance_to_identity(distance: Optional[float]) -> Optional[float]:
|
|
142
278
|
"""Convert distance (0-1) to identity percentage (0-100).
|
|
@@ -217,18 +353,23 @@ class BlastAnalyzer:
|
|
|
217
353
|
min_split: float = 0.005,
|
|
218
354
|
max_lump: float = 0.02,
|
|
219
355
|
target_percentile: int = 100,
|
|
220
|
-
show_progress: bool = True
|
|
356
|
+
show_progress: bool = True,
|
|
357
|
+
find_nearest_cluster: bool = False):
|
|
221
358
|
"""
|
|
222
359
|
Initialize the BLAST analyzer.
|
|
223
360
|
|
|
224
361
|
Args:
|
|
225
362
|
min_split: Minimum distance to split clusters (sequences closer are lumped)
|
|
226
363
|
max_lump: Maximum distance to lump clusters (sequences farther are split)
|
|
227
|
-
target_percentile: Percentile for gap optimization
|
|
364
|
+
target_percentile: Percentile for linkage and gap optimization
|
|
365
|
+
(default: 100 for complete linkage)
|
|
228
366
|
show_progress: Whether to show progress bars
|
|
367
|
+
find_nearest_cluster: Also grow the cluster nearest to the query cluster and
|
|
368
|
+
compare the two (see NearestClusterResult)
|
|
229
369
|
|
|
230
370
|
Raises:
|
|
231
|
-
ValueError: If min_split or max_lump are negative,
|
|
371
|
+
ValueError: If min_split or max_lump are negative, if min_split >= max_lump,
|
|
372
|
+
or if target_percentile is not in 1-100
|
|
232
373
|
"""
|
|
233
374
|
if min_split < 0:
|
|
234
375
|
raise ValueError("min_split must be non-negative")
|
|
@@ -236,11 +377,14 @@ class BlastAnalyzer:
|
|
|
236
377
|
raise ValueError("max_lump must be non-negative")
|
|
237
378
|
if min_split >= max_lump:
|
|
238
379
|
raise ValueError("min_split must be less than max_lump")
|
|
380
|
+
if not 1 <= target_percentile <= 100:
|
|
381
|
+
raise ValueError("target_percentile must be between 1 and 100")
|
|
239
382
|
|
|
240
383
|
self.min_split = min_split
|
|
241
384
|
self.max_lump = max_lump
|
|
242
385
|
self.target_percentile = target_percentile
|
|
243
386
|
self.show_progress = show_progress
|
|
387
|
+
self.find_nearest_cluster = find_nearest_cluster
|
|
244
388
|
|
|
245
389
|
def analyze(self,
|
|
246
390
|
sequences: List[str],
|
|
@@ -333,6 +477,18 @@ class BlastAnalyzer:
|
|
|
333
477
|
identities_to_nearest_non_member
|
|
334
478
|
)
|
|
335
479
|
|
|
480
|
+
# Nearest cluster (optional)
|
|
481
|
+
nearest = None
|
|
482
|
+
if self.find_nearest_cluster:
|
|
483
|
+
nearest = self._analyze_nearest_cluster(
|
|
484
|
+
distance_provider, sequences, headers, sorted(query_cluster_set),
|
|
485
|
+
sorted(remaining_set), warnings
|
|
486
|
+
)
|
|
487
|
+
nearest_members = set(nearest.member_indices) if nearest else None
|
|
488
|
+
for seq_result in sequence_results:
|
|
489
|
+
seq_result.in_nearest_cluster = (seq_result.index in nearest_members
|
|
490
|
+
if nearest_members is not None else None)
|
|
491
|
+
|
|
336
492
|
# Check for warnings
|
|
337
493
|
if len(target_cluster) == 1:
|
|
338
494
|
warnings.append("Query has no close matches")
|
|
@@ -379,7 +535,97 @@ class BlastAnalyzer:
|
|
|
379
535
|
distance_metric="MycoBLAST-adjusted (homopolymer-normalized, indel-normalized)",
|
|
380
536
|
warnings=warnings,
|
|
381
537
|
intra_cluster_histogram=intra_histogram,
|
|
382
|
-
inter_cluster_histogram=inter_histogram
|
|
538
|
+
inter_cluster_histogram=inter_histogram,
|
|
539
|
+
target_percentile=self.target_percentile,
|
|
540
|
+
nearest_cluster_requested=self.find_nearest_cluster,
|
|
541
|
+
nearest_cluster=nearest
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
def _closest_outsider(self,
|
|
545
|
+
distance_provider,
|
|
546
|
+
cluster_indices: List[int],
|
|
547
|
+
outsider_indices: List[int]) -> Tuple[Optional[int], Optional[float]]:
|
|
548
|
+
"""The outsider with the smallest percentile linkage distance to the cluster.
|
|
549
|
+
|
|
550
|
+
Same measure target mode uses to pick the next sequence to add. An outsider with
|
|
551
|
+
any distance that could not be computed (NaN) is never chosen; on a tie the
|
|
552
|
+
earlier input position wins. Returns (None, None) when no outsider qualifies.
|
|
553
|
+
"""
|
|
554
|
+
best_index, best_linkage = None, float('inf')
|
|
555
|
+
for idx in outsider_indices:
|
|
556
|
+
distances = [distance_provider.get_distance(idx, member) for member in cluster_indices]
|
|
557
|
+
if any(np.isnan(d) for d in distances):
|
|
558
|
+
continue
|
|
559
|
+
linkage = float(np.percentile(distances, self.target_percentile))
|
|
560
|
+
if linkage < best_linkage:
|
|
561
|
+
best_index, best_linkage = idx, linkage
|
|
562
|
+
if best_index is None:
|
|
563
|
+
return None, None
|
|
564
|
+
return best_index, best_linkage
|
|
565
|
+
|
|
566
|
+
def _between_identities(self, distance_provider, indices_a: List[int],
|
|
567
|
+
indices_b: List[int]) -> List[float]:
|
|
568
|
+
"""Identity % of every pair with one sequence from each set (NaN pairs skipped)."""
|
|
569
|
+
identities = []
|
|
570
|
+
for a in indices_a:
|
|
571
|
+
for b in indices_b:
|
|
572
|
+
distance = distance_provider.get_distance(a, b)
|
|
573
|
+
if not np.isnan(distance):
|
|
574
|
+
identities.append(_distance_to_identity(distance))
|
|
575
|
+
return identities
|
|
576
|
+
|
|
577
|
+
def _analyze_nearest_cluster(self,
|
|
578
|
+
distance_provider,
|
|
579
|
+
sequences: List[str],
|
|
580
|
+
headers: List[str],
|
|
581
|
+
query_cluster: List[int],
|
|
582
|
+
outsiders: List[int],
|
|
583
|
+
warnings: List[str]) -> Optional[NearestClusterResult]:
|
|
584
|
+
"""Grow the cluster nearest to the query cluster and compare the two.
|
|
585
|
+
|
|
586
|
+
Returns None (with a warning) when there is no sequence outside the query cluster,
|
|
587
|
+
or none with usable distances to it.
|
|
588
|
+
"""
|
|
589
|
+
if not outsiders:
|
|
590
|
+
warnings.append("No sequences outside the query cluster; no nearest cluster")
|
|
591
|
+
return None
|
|
592
|
+
|
|
593
|
+
seed, seed_linkage = self._closest_outsider(distance_provider, query_cluster, outsiders)
|
|
594
|
+
if seed is None:
|
|
595
|
+
warnings.append("No sequence outside the query cluster has usable distances to it; "
|
|
596
|
+
"no nearest cluster")
|
|
597
|
+
return None
|
|
598
|
+
|
|
599
|
+
logger.info("Running target mode clustering for the nearest cluster...")
|
|
600
|
+
subset = SubsetDistanceProvider(distance_provider, outsiders)
|
|
601
|
+
clustering = TargetModeClustering(
|
|
602
|
+
min_split=self.min_split,
|
|
603
|
+
max_lump=self.max_lump,
|
|
604
|
+
target_percentile=self.target_percentile,
|
|
605
|
+
show_progress=self.show_progress
|
|
606
|
+
)
|
|
607
|
+
local_cluster, _, _ = clustering.cluster(
|
|
608
|
+
subset,
|
|
609
|
+
target_indices=[outsiders.index(seed)],
|
|
610
|
+
sequences=[sequences[i] for i in outsiders]
|
|
611
|
+
)
|
|
612
|
+
members = sorted(outsiders[i] for i in local_cluster)
|
|
613
|
+
medoid = self._find_query_cluster_medoid(distance_provider, members)
|
|
614
|
+
|
|
615
|
+
query_pairwise = self._compute_all_intra_cluster_identities(distance_provider, query_cluster)
|
|
616
|
+
between = self._between_identities(distance_provider, query_cluster, members)
|
|
617
|
+
|
|
618
|
+
return NearestClusterResult(
|
|
619
|
+
seed_index=seed,
|
|
620
|
+
seed_id=headers[seed],
|
|
621
|
+
seed_linkage_identity=_distance_to_identity(seed_linkage),
|
|
622
|
+
member_indices=members,
|
|
623
|
+
medoid_index=medoid,
|
|
624
|
+
medoid_id=headers[medoid],
|
|
625
|
+
nearest_pairwise=IdentityDistribution(self._compute_all_intra_cluster_identities(distance_provider, members)),
|
|
626
|
+
query_pairwise=IdentityDistribution(query_pairwise),
|
|
627
|
+
between=IdentityDistribution(between),
|
|
628
|
+
gaps={p: _percentile_gap(query_pairwise, between, p) for p in GAP_PERCENTILES},
|
|
383
629
|
)
|
|
384
630
|
|
|
385
631
|
def _extract_identities_to_query(self,
|
|
@@ -667,7 +913,9 @@ class BlastAnalyzer:
|
|
|
667
913
|
max_lump=self.max_lump,
|
|
668
914
|
normalization_length=query_length,
|
|
669
915
|
distance_metric="MycoBLAST-adjusted (homopolymer-normalized, indel-normalized)",
|
|
670
|
-
warnings=["Insufficient data - need at least 2 sequences for analysis"]
|
|
916
|
+
warnings=["Insufficient data - need at least 2 sequences for analysis"],
|
|
917
|
+
target_percentile=self.target_percentile,
|
|
918
|
+
nearest_cluster_requested=self.find_nearest_cluster
|
|
671
919
|
)
|
|
672
920
|
|
|
673
921
|
def _create_alignment_error_result(self,
|
|
@@ -707,7 +955,9 @@ class BlastAnalyzer:
|
|
|
707
955
|
max_lump=self.max_lump,
|
|
708
956
|
normalization_length=query_length,
|
|
709
957
|
distance_metric="MycoBLAST-adjusted (homopolymer-normalized, indel-normalized)",
|
|
710
|
-
warnings=[f"MSA alignment failed: {error_msg}"]
|
|
958
|
+
warnings=[f"MSA alignment failed: {error_msg}"],
|
|
959
|
+
target_percentile=self.target_percentile,
|
|
960
|
+
nearest_cluster_requested=self.find_nearest_cluster
|
|
711
961
|
)
|
|
712
962
|
|
|
713
963
|
|
|
@@ -757,7 +1007,7 @@ def format_text_output(result: BlastAnalysisResult) -> str:
|
|
|
757
1007
|
lines.append("-" * 85)
|
|
758
1008
|
|
|
759
1009
|
for seq in result.sequences:
|
|
760
|
-
in_cluster = "Yes" if seq.in_query_cluster else "No"
|
|
1010
|
+
in_cluster = "Yes" if seq.in_query_cluster else ("Nearest" if seq.in_nearest_cluster else "No")
|
|
761
1011
|
id_query = f"{seq.identity_to_query_normalized:.2f}%" if seq.identity_to_query_normalized is not None else "N/A"
|
|
762
1012
|
id_medoid = f"{seq.identity_to_medoid_normalized:.2f}%" if seq.identity_to_medoid_normalized is not None else "N/A"
|
|
763
1013
|
id_non_mem = f"{seq.identity_to_nearest_non_member_normalized:.2f}%" if seq.identity_to_nearest_non_member_normalized is not None else "N/A"
|
|
@@ -765,6 +1015,9 @@ def format_text_output(result: BlastAnalysisResult) -> str:
|
|
|
765
1015
|
|
|
766
1016
|
lines.append("")
|
|
767
1017
|
|
|
1018
|
+
if result.nearest_cluster_requested:
|
|
1019
|
+
lines.extend(_format_nearest_cluster_text(result))
|
|
1020
|
+
|
|
768
1021
|
# Diagnostics
|
|
769
1022
|
lines.append("-" * 40)
|
|
770
1023
|
lines.append("Diagnostics")
|
|
@@ -772,6 +1025,8 @@ def format_text_output(result: BlastAnalysisResult) -> str:
|
|
|
772
1025
|
lines.append(f"Method: {result.method}")
|
|
773
1026
|
lines.append(f"min_split: {result.min_split}")
|
|
774
1027
|
lines.append(f"max_lump: {result.max_lump}")
|
|
1028
|
+
if result.target_percentile is not None:
|
|
1029
|
+
lines.append(f"target_percentile: {result.target_percentile}")
|
|
775
1030
|
lines.append(f"Normalization length: {result.normalization_length} bp")
|
|
776
1031
|
lines.append(f"Identity metric: {result.distance_metric}")
|
|
777
1032
|
|
|
@@ -786,12 +1041,56 @@ def format_text_output(result: BlastAnalysisResult) -> str:
|
|
|
786
1041
|
return "\n".join(lines)
|
|
787
1042
|
|
|
788
1043
|
|
|
1044
|
+
def _format_nearest_cluster_text(result: BlastAnalysisResult) -> List[str]:
|
|
1045
|
+
"""Text section for the nearest cluster."""
|
|
1046
|
+
lines = ["-" * 40, "Nearest Cluster", "-" * 40]
|
|
1047
|
+
nearest = result.nearest_cluster
|
|
1048
|
+
if nearest is None:
|
|
1049
|
+
lines.append("No nearest cluster (see warnings)")
|
|
1050
|
+
lines.append("")
|
|
1051
|
+
return lines
|
|
1052
|
+
|
|
1053
|
+
def fmt(val: Optional[float]) -> str:
|
|
1054
|
+
return f"{val:.2f}%" if val is not None else "N/A"
|
|
1055
|
+
|
|
1056
|
+
linkage = f"P{result.target_percentile} " if result.target_percentile is not None else ""
|
|
1057
|
+
lines.append(f"Seed: {nearest.seed_id} (index {nearest.seed_index}), "
|
|
1058
|
+
f"{fmt(nearest.seed_linkage_identity)} identity to the query cluster ({linkage}linkage)")
|
|
1059
|
+
lines.append(f"Nearest cluster size: {len(nearest.member_indices)}")
|
|
1060
|
+
lines.append(f"Nearest cluster medoid: {nearest.medoid_id} (index {nearest.medoid_index})")
|
|
1061
|
+
for label, dist in (("Within query cluster", nearest.query_pairwise),
|
|
1062
|
+
("Within nearest cluster", nearest.nearest_pairwise),
|
|
1063
|
+
("Between clusters", nearest.between)):
|
|
1064
|
+
s = dist.summary()
|
|
1065
|
+
if s["count"]:
|
|
1066
|
+
lines.append(f"{label}: min={fmt(s['min'])}, median={fmt(s['median'])}, "
|
|
1067
|
+
f"max={fmt(s['max'])} ({s['count']} pairs)")
|
|
1068
|
+
lines.append("Barcode gap, query cluster vs nearest cluster:")
|
|
1069
|
+
for pct, gap in nearest.gaps.items():
|
|
1070
|
+
if gap is None:
|
|
1071
|
+
lines.append(f" P{pct}: N/A")
|
|
1072
|
+
continue
|
|
1073
|
+
status = "gap" if gap.gap_exists else "no gap"
|
|
1074
|
+
lines.append(f" P{pct}: {gap.gap_size_percent:.2f}% ({status}; query cluster "
|
|
1075
|
+
f"P{100 - pct} {fmt(gap.intra_lower_identity)} vs between P{pct} "
|
|
1076
|
+
f"{fmt(gap.inter_upper_identity)})")
|
|
1077
|
+
lines.append("")
|
|
1078
|
+
return lines
|
|
1079
|
+
|
|
1080
|
+
|
|
789
1081
|
def format_tsv_output(result: BlastAnalysisResult) -> str:
|
|
790
|
-
"""Format analysis result as TSV (tab-separated values).
|
|
1082
|
+
"""Format analysis result as TSV (tab-separated values).
|
|
1083
|
+
|
|
1084
|
+
An in_nearest_cluster column is appended when the nearest cluster was requested.
|
|
1085
|
+
"""
|
|
791
1086
|
lines = []
|
|
1087
|
+
nearest_column = result.nearest_cluster_requested
|
|
792
1088
|
|
|
793
1089
|
# Header
|
|
794
|
-
|
|
1090
|
+
header = "index\tid\tin_query_cluster\tidentity_to_query\tidentity_to_query_normalized\tidentity_to_medoid_normalized\tidentity_to_nearest_non_member_normalized"
|
|
1091
|
+
if nearest_column:
|
|
1092
|
+
header += "\tin_nearest_cluster"
|
|
1093
|
+
lines.append(header)
|
|
795
1094
|
|
|
796
1095
|
# Data rows
|
|
797
1096
|
for seq in result.sequences:
|
|
@@ -800,6 +1099,9 @@ def format_tsv_output(result: BlastAnalysisResult) -> str:
|
|
|
800
1099
|
identity_norm = f"{seq.identity_to_query_normalized:.2f}" if seq.identity_to_query_normalized is not None else ""
|
|
801
1100
|
identity_medoid = f"{seq.identity_to_medoid_normalized:.2f}" if seq.identity_to_medoid_normalized is not None else ""
|
|
802
1101
|
identity_non_mem = f"{seq.identity_to_nearest_non_member_normalized:.2f}" if seq.identity_to_nearest_non_member_normalized is not None else ""
|
|
803
|
-
|
|
1102
|
+
row = f"{seq.index}\t{seq.id}\t{in_cluster}\t{identity}\t{identity_norm}\t{identity_medoid}\t{identity_non_mem}"
|
|
1103
|
+
if nearest_column:
|
|
1104
|
+
row += "\t" + ("" if seq.in_nearest_cluster is None else ("true" if seq.in_nearest_cluster else "false"))
|
|
1105
|
+
lines.append(row)
|
|
804
1106
|
|
|
805
1107
|
return "\n".join(lines)
|
|
@@ -86,10 +86,15 @@ Examples:
|
|
|
86
86
|
# Adjust clustering thresholds
|
|
87
87
|
gaphack-blast blast_results.fasta --min-split 0.003 --max-lump 0.03
|
|
88
88
|
|
|
89
|
+
# Also find the cluster nearest to the query's and compare the two
|
|
90
|
+
gaphack-blast blast_results.fasta --nearest-cluster
|
|
91
|
+
|
|
89
92
|
Output fields:
|
|
90
93
|
- in_query_cluster: Whether the sequence clusters with the query
|
|
91
94
|
- identity_to_query: MycoBLAST-adjusted identity (e.g., 99.63%)
|
|
92
95
|
- gap_size_percent: Barcode gap magnitude in percentage points
|
|
96
|
+
- nearest_cluster (with --nearest-cluster): the nearest cluster, and the gap
|
|
97
|
+
between it and the query cluster at P90, P95 and P100
|
|
93
98
|
"""
|
|
94
99
|
)
|
|
95
100
|
|
|
@@ -125,6 +130,19 @@ Output fields:
|
|
|
125
130
|
default=0.02,
|
|
126
131
|
help='Maximum distance to lump clusters (default: 0.02 = 98%% identity)'
|
|
127
132
|
)
|
|
133
|
+
parser.add_argument(
|
|
134
|
+
'--target-percentile',
|
|
135
|
+
type=int,
|
|
136
|
+
default=100,
|
|
137
|
+
help='Percentile for linkage and gap optimization when growing a cluster '
|
|
138
|
+
'(default: 100 = complete linkage; gaphack --target uses 95)'
|
|
139
|
+
)
|
|
140
|
+
parser.add_argument(
|
|
141
|
+
'--nearest-cluster',
|
|
142
|
+
action='store_true',
|
|
143
|
+
help='Also grow the cluster nearest to the query cluster from the closest '
|
|
144
|
+
'outside sequence, and report the gap between the two at P90, P95 and P100'
|
|
145
|
+
)
|
|
128
146
|
|
|
129
147
|
# Verbosity
|
|
130
148
|
parser.add_argument(
|
|
@@ -171,7 +189,9 @@ Output fields:
|
|
|
171
189
|
analyzer = BlastAnalyzer(
|
|
172
190
|
min_split=args.min_split,
|
|
173
191
|
max_lump=args.max_lump,
|
|
174
|
-
|
|
192
|
+
target_percentile=args.target_percentile,
|
|
193
|
+
show_progress=not args.quiet,
|
|
194
|
+
find_nearest_cluster=args.nearest_cluster
|
|
175
195
|
)
|
|
176
196
|
|
|
177
197
|
result = analyzer.analyze(sequences, headers)
|
|
@@ -201,6 +221,11 @@ Output fields:
|
|
|
201
221
|
logging.info(f"Barcode gap found: {result.gap_size_percent:.2f}%")
|
|
202
222
|
else:
|
|
203
223
|
logging.info("No barcode gap found")
|
|
224
|
+
if args.nearest_cluster and result.nearest_cluster is not None:
|
|
225
|
+
gap = result.nearest_cluster.gaps.get(95)
|
|
226
|
+
gap_text = f", P95 gap {gap.gap_size_percent:.2f}%" if gap else ""
|
|
227
|
+
logging.info(f"Nearest cluster: {len(result.nearest_cluster.member_indices)} sequences "
|
|
228
|
+
f"(seed {result.nearest_cluster.seed_id}){gap_text}")
|
|
204
229
|
|
|
205
230
|
except KeyboardInterrupt:
|
|
206
231
|
logging.info("Interrupted by user")
|
|
@@ -194,3 +194,26 @@ class PrecomputedDistanceProvider(DistanceProvider):
|
|
|
194
194
|
return self.distance_matrix
|
|
195
195
|
|
|
196
196
|
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class SubsetDistanceProvider(DistanceProvider):
|
|
200
|
+
"""A view of another provider restricted to some of its sequences.
|
|
201
|
+
|
|
202
|
+
Local index i is the base provider's index ``indices[i]``. Distances come from the
|
|
203
|
+
base provider (and its cache), so a subset clustered on its own keeps the distances
|
|
204
|
+
of the full alignment rather than realigning.
|
|
205
|
+
"""
|
|
206
|
+
|
|
207
|
+
def __init__(self, base: DistanceProvider, indices: List[int]):
|
|
208
|
+
self.base = base
|
|
209
|
+
self.indices = list(indices)
|
|
210
|
+
self.n = len(self.indices)
|
|
211
|
+
|
|
212
|
+
def get_distance(self, idx1: int, idx2: int) -> float:
|
|
213
|
+
return self.base.get_distance(self.indices[idx1], self.indices[idx2])
|
|
214
|
+
|
|
215
|
+
def get_distances_from_sequence(self, idx: int, targets: Set[int]) -> Dict[int, float]:
|
|
216
|
+
return {target_idx: self.get_distance(idx, target_idx) for target_idx in targets}
|
|
217
|
+
|
|
218
|
+
def ensure_distances_computed(self, indices: Set[int]) -> None:
|
|
219
|
+
self.base.ensure_distances_computed({self.indices[i] for i in indices})
|
|
@@ -63,6 +63,11 @@ class TargetModeClustering:
|
|
|
63
63
|
target_indices: List of sequence indices that form the initial seed cluster
|
|
64
64
|
sequences: List of sequences (required if distance_provider is not a DistanceProvider)
|
|
65
65
|
|
|
66
|
+
The cluster grows one sequence at a time, closest first, until the closest
|
|
67
|
+
remaining sequence is farther than max_lump. The result is the configuration with
|
|
68
|
+
the widest gap among those that include every merge made within min_split (ties
|
|
69
|
+
go to the larger cluster).
|
|
70
|
+
|
|
66
71
|
Returns:
|
|
67
72
|
Tuple of (target_cluster, remaining_sequences, clustering_history) where:
|
|
68
73
|
- target_cluster is List[int] of indices in the final target cluster
|
|
@@ -210,9 +215,16 @@ class TargetModeClustering:
|
|
|
210
215
|
'merge_reason': merge_reason
|
|
211
216
|
})
|
|
212
217
|
|
|
213
|
-
# Track best configuration (prefer larger clusters when gaps are equal)
|
|
218
|
+
# Track best configuration (prefer larger clusters when gaps are equal).
|
|
219
|
+
# A merge within min_split is mandatory: sequences that close always belong
|
|
220
|
+
# together, so no configuration that leaves this sequence out may be chosen.
|
|
221
|
+
# The best configuration therefore restarts here, whatever its gap; only this
|
|
222
|
+
# and later configurations remain candidates. Full gapHACk gets the same
|
|
223
|
+
# effect by finishing all merges within min_split before it tracks gaps.
|
|
214
224
|
update_best = False
|
|
215
|
-
if
|
|
225
|
+
if closest_distance <= self.min_split:
|
|
226
|
+
update_best = True
|
|
227
|
+
elif current_gap > best_config['gap_size']:
|
|
216
228
|
# Strictly better gap
|
|
217
229
|
update_best = True
|
|
218
230
|
elif current_gap == best_config['gap_size'] and len(target_cluster) > len(best_config['target_cluster']):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gaphack
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.2
|
|
4
4
|
Summary: Gap-Optimized Hierarchical Agglomerative Clustering for DNA barcoding
|
|
5
5
|
Author-email: Josh Walker <joshowalker@yahoo.com>
|
|
6
6
|
License: BSD-2-Clause
|
|
@@ -280,6 +280,8 @@ gaphack input.fasta --target seeds.fasta -o target_results
|
|
|
280
280
|
|
|
281
281
|
Target mode focuses on growing one cluster from the provided seed sequences, making it suitable for cases where you want to extract sequences similar to specific targets without attempting to cluster all remaining sequences.
|
|
282
282
|
|
|
283
|
+
The cluster grows one sequence at a time, closest first, up to `--max-lump`. The result is the size with the widest gap among those that include every merge made within `--min-split`: sequences that close to the cluster always join it.
|
|
284
|
+
|
|
283
285
|
### Iterative Refinement (`gaphack-refine`)
|
|
284
286
|
|
|
285
287
|
The `gaphack-refine` tool optimizes cluster boundaries through iterative neighborhood-based refinement. It works with any input clustering (vsearch, CD-HIT, MMseqs2, or gaphack output).
|
|
@@ -420,6 +422,9 @@ gaphack-blast sequences.fasta --format text
|
|
|
420
422
|
|
|
421
423
|
# Tab-separated for spreadsheets
|
|
422
424
|
gaphack-blast sequences.fasta --format tsv
|
|
425
|
+
|
|
426
|
+
# Also find the cluster nearest to the query's and compare the two
|
|
427
|
+
gaphack-blast sequences.fasta --nearest-cluster
|
|
423
428
|
```
|
|
424
429
|
|
|
425
430
|
**Key output fields:**
|
|
@@ -427,6 +432,8 @@ gaphack-blast sequences.fasta --format tsv
|
|
|
427
432
|
- `identity_to_query`: MycoBLAST-adjusted identity percentage
|
|
428
433
|
- `barcode_gap_found`: Whether a clear species boundary was detected
|
|
429
434
|
- `gap_size_percent`: Magnitude of the barcode gap
|
|
435
|
+
- `nearest_cluster` (with `--nearest-cluster`): the cluster grown from the closest
|
|
436
|
+
sequence outside the query cluster, and the barcode gap between the two at P90, P95 and P100
|
|
430
437
|
|
|
431
438
|
For detailed field documentation and integration patterns, see [docs/blast_analysis_integration.md](docs/blast_analysis_integration.md).
|
|
432
439
|
|
|
@@ -969,3 +969,162 @@ class TestHistograms:
|
|
|
969
969
|
assert result.inter_cluster_histogram is not None
|
|
970
970
|
# One entry per cluster member
|
|
971
971
|
assert sum(result.inter_cluster_histogram.counts) == result.query_cluster_size
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
from gaphack.distance_providers import PrecomputedDistanceProvider
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
class _MatrixProvider(PrecomputedDistanceProvider):
|
|
978
|
+
"""A real DistanceProvider over a matrix, with the extras BlastAnalyzer reads."""
|
|
979
|
+
normalization_length = 100
|
|
980
|
+
|
|
981
|
+
def get_distance_detailed(self, i, j):
|
|
982
|
+
d = float(self.distance_matrix[i][j])
|
|
983
|
+
return MSADistanceResult(distance_normalized=d, distance_pairwise=d,
|
|
984
|
+
mismatches=int(round(d * 100)), pairwise_overlap=100, is_valid=True)
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
def _three_groups_matrix():
|
|
988
|
+
"""Query group 0-3, nearest group 4-7 (0.03 away), far pair 8-9 (0.08 / 0.06 away).
|
|
989
|
+
Pairs within a group are 0.004 apart."""
|
|
990
|
+
groups = [range(0, 4), range(4, 8), range(8, 10)]
|
|
991
|
+
between = {(0, 1): 0.03, (0, 2): 0.08, (1, 2): 0.06}
|
|
992
|
+
m = np.zeros((10, 10))
|
|
993
|
+
for gi, a in enumerate(groups):
|
|
994
|
+
for gj, b in enumerate(groups):
|
|
995
|
+
for i in a:
|
|
996
|
+
for j in b:
|
|
997
|
+
if i != j:
|
|
998
|
+
m[i][j] = 0.004 if gi == gj else between[tuple(sorted((gi, gj)))]
|
|
999
|
+
return m
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
class TestNearestCluster:
|
|
1003
|
+
"""Optional nearest cluster: grown from the closest outsider, compared at P90/P95/P100."""
|
|
1004
|
+
|
|
1005
|
+
def _analyze(self, matrix, **kwargs):
|
|
1006
|
+
n = len(matrix)
|
|
1007
|
+
analyzer = BlastAnalyzer(min_split=0.005, max_lump=0.02, show_progress=False, **kwargs)
|
|
1008
|
+
with patch('gaphack.blast_analysis.MSACachedDistanceProvider',
|
|
1009
|
+
return_value=_MatrixProvider(matrix)):
|
|
1010
|
+
return analyzer.analyze([f"SEQ{i}" for i in range(n)], [f"seq{i}" for i in range(n)])
|
|
1011
|
+
|
|
1012
|
+
def test_nearest_cluster_and_gaps(self):
|
|
1013
|
+
result = self._analyze(_three_groups_matrix(), find_nearest_cluster=True)
|
|
1014
|
+
|
|
1015
|
+
assert [s.index for s in result.sequences if s.in_query_cluster] == [0, 1, 2, 3]
|
|
1016
|
+
nearest = result.nearest_cluster
|
|
1017
|
+
assert nearest is not None
|
|
1018
|
+
assert nearest.seed_index == 4 # all of 4-7 tie; the earliest wins
|
|
1019
|
+
assert nearest.member_indices == [4, 5, 6, 7]
|
|
1020
|
+
assert nearest.seed_linkage_identity == pytest.approx(97.0)
|
|
1021
|
+
assert [s.in_nearest_cluster for s in result.sequences] == [False] * 4 + [True] * 4 + [False] * 2
|
|
1022
|
+
|
|
1023
|
+
assert len(nearest.query_pairwise.values) == 6
|
|
1024
|
+
assert len(nearest.nearest_pairwise.values) == 6
|
|
1025
|
+
assert len(nearest.between.values) == 16
|
|
1026
|
+
for pct in (90, 95, 100):
|
|
1027
|
+
gap = nearest.gaps[pct]
|
|
1028
|
+
assert gap.intra_lower_identity == pytest.approx(99.6)
|
|
1029
|
+
assert gap.inter_upper_identity == pytest.approx(97.0)
|
|
1030
|
+
assert gap.gap_size_percent == pytest.approx(2.6)
|
|
1031
|
+
assert gap.gap_exists
|
|
1032
|
+
|
|
1033
|
+
d = result.to_dict()
|
|
1034
|
+
assert d["nearest_cluster"]["size"] == 4
|
|
1035
|
+
assert d["nearest_cluster"]["seed_id"] == "seq4"
|
|
1036
|
+
assert d["nearest_cluster"]["comparison"]["gaps"]["p95"]["gap_size_percent"] == 2.6
|
|
1037
|
+
assert d["nearest_cluster"]["comparison"]["between_clusters_identity"]["count"] == 16
|
|
1038
|
+
assert [s["in_nearest_cluster"] for s in d["sequences"]][4] is True
|
|
1039
|
+
assert d["diagnostics"]["histograms"]["between_clusters"] is not None
|
|
1040
|
+
assert d["diagnostics"]["target_percentile"] == 100
|
|
1041
|
+
assert json.loads(json.dumps(d)) == d
|
|
1042
|
+
|
|
1043
|
+
tsv = format_tsv_output(result).split("\n")
|
|
1044
|
+
assert tsv[0].endswith("\tin_nearest_cluster")
|
|
1045
|
+
assert tsv[5].split("\t")[-1] == "true"
|
|
1046
|
+
assert tsv[1].split("\t")[-1] == "false"
|
|
1047
|
+
|
|
1048
|
+
text = format_text_output(result)
|
|
1049
|
+
assert "Nearest Cluster" in text
|
|
1050
|
+
assert "Nearest cluster size: 4" in text
|
|
1051
|
+
assert "P95: 2.60% (gap" in text
|
|
1052
|
+
|
|
1053
|
+
def test_gap_percentiles_use_the_tails(self):
|
|
1054
|
+
# One loose pair within the query cluster and one close pair between the clusters:
|
|
1055
|
+
# P100 compares those two extremes, P90 cuts them off.
|
|
1056
|
+
from gaphack.blast_analysis import _percentile_gap
|
|
1057
|
+
intra = [98.8] + [99.6] * 9
|
|
1058
|
+
inter = [98.9] + [97.0] * 19
|
|
1059
|
+
p100 = _percentile_gap(intra, inter, 100)
|
|
1060
|
+
assert p100.intra_lower_identity == pytest.approx(98.8)
|
|
1061
|
+
assert p100.inter_upper_identity == pytest.approx(98.9)
|
|
1062
|
+
assert p100.gap_size_percent == pytest.approx(-0.1)
|
|
1063
|
+
assert not p100.gap_exists
|
|
1064
|
+
p90 = _percentile_gap(intra, inter, 90)
|
|
1065
|
+
assert p90.intra_lower_identity == pytest.approx(np.percentile(intra, 10))
|
|
1066
|
+
assert p90.inter_upper_identity == pytest.approx(np.percentile(inter, 90))
|
|
1067
|
+
assert p90.gap_exists
|
|
1068
|
+
assert _percentile_gap([], inter, 95) is None
|
|
1069
|
+
assert _percentile_gap(intra, [], 95) is None
|
|
1070
|
+
|
|
1071
|
+
def test_no_outsiders(self):
|
|
1072
|
+
m = np.full((3, 3), 0.004)
|
|
1073
|
+
np.fill_diagonal(m, 0.0)
|
|
1074
|
+
result = self._analyze(m, find_nearest_cluster=True)
|
|
1075
|
+
assert result.query_cluster_size == 3
|
|
1076
|
+
assert result.nearest_cluster is None
|
|
1077
|
+
assert any("no nearest cluster" in w for w in result.warnings)
|
|
1078
|
+
d = result.to_dict()
|
|
1079
|
+
assert d["nearest_cluster"] is None
|
|
1080
|
+
assert all(s["in_nearest_cluster"] is None for s in d["sequences"])
|
|
1081
|
+
assert format_tsv_output(result).split("\n")[1].endswith("\t")
|
|
1082
|
+
|
|
1083
|
+
def test_not_requested_leaves_output_unchanged(self):
|
|
1084
|
+
result = self._analyze(_three_groups_matrix())
|
|
1085
|
+
assert result.nearest_cluster is None
|
|
1086
|
+
d = result.to_dict()
|
|
1087
|
+
assert "nearest_cluster" not in d
|
|
1088
|
+
assert all("in_nearest_cluster" not in s for s in d["sequences"])
|
|
1089
|
+
assert set(d["diagnostics"]["histograms"]) == {"intra_cluster", "inter_cluster"}
|
|
1090
|
+
assert not format_tsv_output(result).split("\n")[0].endswith("in_nearest_cluster")
|
|
1091
|
+
assert "Nearest Cluster" not in format_text_output(result)
|
|
1092
|
+
|
|
1093
|
+
def test_target_percentile_validated(self):
|
|
1094
|
+
with pytest.raises(ValueError):
|
|
1095
|
+
BlastAnalyzer(target_percentile=0)
|
|
1096
|
+
with pytest.raises(ValueError):
|
|
1097
|
+
BlastAnalyzer(target_percentile=101)
|
|
1098
|
+
|
|
1099
|
+
def test_real_sequences(self):
|
|
1100
|
+
fasta_path = TEST_DATA_DIR / "russula_diverse_50.fasta"
|
|
1101
|
+
if not fasta_path.exists():
|
|
1102
|
+
pytest.skip(f"Test data not found: {fasta_path}")
|
|
1103
|
+
records = list(SeqIO.parse(fasta_path, "fasta"))[:30]
|
|
1104
|
+
sequences = [str(r.seq).upper() for r in records]
|
|
1105
|
+
headers = [r.id for r in records]
|
|
1106
|
+
result = BlastAnalyzer(show_progress=False, find_nearest_cluster=True).analyze(sequences, headers)
|
|
1107
|
+
|
|
1108
|
+
query = {s.index for s in result.sequences if s.in_query_cluster}
|
|
1109
|
+
nearest = result.nearest_cluster
|
|
1110
|
+
assert nearest is not None
|
|
1111
|
+
members = set(nearest.member_indices)
|
|
1112
|
+
assert nearest.seed_index in members
|
|
1113
|
+
assert not (members & query)
|
|
1114
|
+
assert len(nearest.between.values) <= len(query) * len(members)
|
|
1115
|
+
assert set(nearest.gaps) == {90, 95, 100}
|
|
1116
|
+
|
|
1117
|
+
def test_cli_nearest_cluster(self, tmp_path, capsys):
|
|
1118
|
+
from gaphack import blast_cli
|
|
1119
|
+
fasta_path = TEST_DATA_DIR / "russula_diverse_50.fasta"
|
|
1120
|
+
if not fasta_path.exists():
|
|
1121
|
+
pytest.skip(f"Test data not found: {fasta_path}")
|
|
1122
|
+
records = list(SeqIO.parse(fasta_path, "fasta"))[:20]
|
|
1123
|
+
small = tmp_path / "hits.fasta"
|
|
1124
|
+
SeqIO.write(records, small, "fasta")
|
|
1125
|
+
with patch("sys.argv", ["gaphack-blast", str(small), "--nearest-cluster",
|
|
1126
|
+
"--target-percentile", "95", "-q"]):
|
|
1127
|
+
blast_cli.main()
|
|
1128
|
+
d = json.loads(capsys.readouterr().out)
|
|
1129
|
+
assert d["diagnostics"]["target_percentile"] == 95
|
|
1130
|
+
assert "nearest_cluster" in d
|
|
@@ -158,3 +158,20 @@ class TestMSACachedDistanceProvider:
|
|
|
158
158
|
assert matrix.shape == (3, 3)
|
|
159
159
|
assert np.all(np.diag(matrix) == 0.0) # Diagonal should be zero
|
|
160
160
|
assert matrix[0, 1] == matrix[1, 0] # Should be symmetric
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class TestSubsetDistanceProvider:
|
|
164
|
+
"""A subset view maps local indices to the base provider's."""
|
|
165
|
+
|
|
166
|
+
def test_maps_indices_to_base(self):
|
|
167
|
+
from gaphack.distance_providers import PrecomputedDistanceProvider, SubsetDistanceProvider
|
|
168
|
+
matrix = np.array([
|
|
169
|
+
[0.0, 0.1, 0.2, 0.3],
|
|
170
|
+
[0.1, 0.0, 0.4, 0.5],
|
|
171
|
+
[0.2, 0.4, 0.0, 0.6],
|
|
172
|
+
[0.3, 0.5, 0.6, 0.0],
|
|
173
|
+
])
|
|
174
|
+
subset = SubsetDistanceProvider(PrecomputedDistanceProvider(matrix), [3, 1])
|
|
175
|
+
assert subset.n == 2
|
|
176
|
+
assert subset.get_distance(0, 1) == 0.5
|
|
177
|
+
assert subset.get_distances_from_sequence(0, {1}) == {1: 0.5}
|
|
@@ -283,4 +283,105 @@ class TestTargetModeClustering:
|
|
|
283
283
|
# Target cluster size should increase with each step
|
|
284
284
|
sizes = [entry['target_cluster_size'] for entry in gap_history]
|
|
285
285
|
for i in range(1, len(sizes)):
|
|
286
|
-
assert sizes[i] == sizes[i-1] + 1
|
|
286
|
+
assert sizes[i] == sizes[i-1] + 1
|
|
287
|
+
|
|
288
|
+
def _best_with_mandatory_merges(seed, gap_history, min_split):
|
|
289
|
+
"""Reference rule, computed from the history alone: the widest-gap configuration at or
|
|
290
|
+
after the last merge within min_split (ties to the larger cluster). Returns None when no
|
|
291
|
+
merge was within min_split."""
|
|
292
|
+
last_mandatory = None
|
|
293
|
+
for position, step in enumerate(gap_history):
|
|
294
|
+
if step['merge_distance'] <= min_split:
|
|
295
|
+
last_mandatory = position
|
|
296
|
+
if last_mandatory is None:
|
|
297
|
+
return None
|
|
298
|
+
best = last_mandatory
|
|
299
|
+
for position in range(last_mandatory, len(gap_history)):
|
|
300
|
+
if gap_history[position]['gap_size'] >= gap_history[best]['gap_size']:
|
|
301
|
+
best = position
|
|
302
|
+
return set(seed) | {step['merged_sequence'] for step in gap_history[:best + 1]}
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
class TestMandatoryMerges:
|
|
306
|
+
"""Merges within min_split are mandatory: the chosen cluster always includes them."""
|
|
307
|
+
|
|
308
|
+
@staticmethod
|
|
309
|
+
def _sequences(n):
|
|
310
|
+
return [f"SEQ{i}" for i in range(n)]
|
|
311
|
+
|
|
312
|
+
def test_mandatory_merge_kept_when_it_narrows_the_gap(self):
|
|
313
|
+
# 1 is 0.001 from the target and 2 exactly min_split from both: two mandatory merges.
|
|
314
|
+
# The gap is wider after the first (0.005 - 0.001) than after the second
|
|
315
|
+
# (0.008 - 0.005), so choosing by gap alone would leave out sequence 2.
|
|
316
|
+
distance_matrix = np.array([
|
|
317
|
+
[0.0, 0.001, 0.005, 0.008],
|
|
318
|
+
[0.001, 0.0, 0.005, 0.008],
|
|
319
|
+
[0.005, 0.005, 0.0, 0.008],
|
|
320
|
+
[0.008, 0.008, 0.008, 0.0],
|
|
321
|
+
])
|
|
322
|
+
clustering = TargetModeClustering(min_split=0.005, max_lump=0.006,
|
|
323
|
+
target_percentile=100, show_progress=False)
|
|
324
|
+
cluster, remaining, metrics = clustering.cluster(
|
|
325
|
+
PrecomputedDistanceProvider(distance_matrix), [0], self._sequences(4))
|
|
326
|
+
|
|
327
|
+
history = metrics['gap_history']
|
|
328
|
+
assert [step['merged_sequence'] for step in history] == [1, 2]
|
|
329
|
+
assert history[0]['gap_size'] > history[1]['gap_size']
|
|
330
|
+
assert sorted(cluster) == [0, 1, 2]
|
|
331
|
+
assert remaining == [3]
|
|
332
|
+
assert sorted(metrics['best_config']['target_cluster']) == [0, 1, 2]
|
|
333
|
+
assert metrics['best_config']['merge_distance'] == pytest.approx(0.005)
|
|
334
|
+
|
|
335
|
+
def test_without_mandatory_merges_the_widest_gap_wins(self):
|
|
336
|
+
# Nothing within min_split: the choice is by gap alone, as before.
|
|
337
|
+
distance_matrix = np.array([
|
|
338
|
+
[0.0, 0.01, 0.03],
|
|
339
|
+
[0.01, 0.0, 0.03],
|
|
340
|
+
[0.03, 0.03, 0.0],
|
|
341
|
+
])
|
|
342
|
+
clustering = TargetModeClustering(min_split=0.005, max_lump=0.02,
|
|
343
|
+
target_percentile=100, show_progress=False)
|
|
344
|
+
cluster, remaining, _ = clustering.cluster(
|
|
345
|
+
PrecomputedDistanceProvider(distance_matrix), [0], self._sequences(3))
|
|
346
|
+
assert sorted(cluster) == [0, 1]
|
|
347
|
+
assert remaining == [2]
|
|
348
|
+
|
|
349
|
+
def test_random_matrices_match_reference_rule(self):
|
|
350
|
+
"""On random clustered data (percentile linkage, so merges within min_split can
|
|
351
|
+
follow wider ones), the result is the reference rule's, and always contains every
|
|
352
|
+
sequence merged within min_split."""
|
|
353
|
+
rng = np.random.default_rng(20261119)
|
|
354
|
+
min_split, max_lump = 0.005, 0.02
|
|
355
|
+
differs_from_gap_only = 0
|
|
356
|
+
checked = 0
|
|
357
|
+
for _ in range(300):
|
|
358
|
+
n = int(rng.integers(8, 30))
|
|
359
|
+
centers = rng.normal(0.0, 0.012, size=(int(rng.integers(1, 4)), 2))
|
|
360
|
+
points = centers[rng.integers(0, len(centers), size=n)] + rng.normal(0.0, 0.003, size=(n, 2))
|
|
361
|
+
distance_matrix = np.sqrt(((points[:, None, :] - points[None, :, :]) ** 2).sum(-1))
|
|
362
|
+
percentile = int(rng.choice([90, 95, 100]))
|
|
363
|
+
clustering = TargetModeClustering(min_split=min_split, max_lump=max_lump,
|
|
364
|
+
target_percentile=percentile, show_progress=False)
|
|
365
|
+
cluster, _, metrics = clustering.cluster(
|
|
366
|
+
PrecomputedDistanceProvider(distance_matrix), [0], self._sequences(n))
|
|
367
|
+
history = metrics['gap_history']
|
|
368
|
+
|
|
369
|
+
mandatory = {step['merged_sequence'] for step in history
|
|
370
|
+
if step['merge_distance'] <= min_split}
|
|
371
|
+
assert mandatory <= set(cluster)
|
|
372
|
+
|
|
373
|
+
expected = _best_with_mandatory_merges([0], history, min_split)
|
|
374
|
+
if expected is None:
|
|
375
|
+
continue
|
|
376
|
+
checked += 1
|
|
377
|
+
assert set(cluster) == expected
|
|
378
|
+
|
|
379
|
+
# Gap-only choice (the pre-0.7.2 rule), to confirm the data exercises the fix
|
|
380
|
+
gaps = [0.0] + [step['gap_size'] for step in history]
|
|
381
|
+
best = max(range(len(gaps)), key=lambda i: (gaps[i], i))
|
|
382
|
+
if best < len(history) and best <= max(i for i, s in enumerate(history)
|
|
383
|
+
if s['merge_distance'] <= min_split):
|
|
384
|
+
differs_from_gap_only += 1
|
|
385
|
+
|
|
386
|
+
assert checked > 100
|
|
387
|
+
assert differs_from_gap_only > 0
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""The package version: gaphack/version.py is the source in code, and matches pyproject.toml."""
|
|
2
|
+
import re
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import gaphack
|
|
6
|
+
from gaphack import version
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_version_sources_agree():
|
|
10
|
+
pyproject = (Path(__file__).parent.parent / "pyproject.toml").read_text()
|
|
11
|
+
declared = re.search(r'^version = "([^"]+)"', pyproject, re.M).group(1)
|
|
12
|
+
assert version.__version__ == declared
|
|
13
|
+
assert gaphack.__version__ == declared
|
gaphack-0.7.1/gaphack/version.py
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|