gaphack 0.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gaphack/__init__.py +54 -0
- gaphack/analyze.py +356 -0
- gaphack/analyze_cli.py +352 -0
- gaphack/blast_analysis.py +805 -0
- gaphack/blast_cli.py +217 -0
- gaphack/blast_neighborhood.py +354 -0
- gaphack/cli.py +379 -0
- gaphack/cluster_graph.py +733 -0
- gaphack/cluster_refinement.py +1713 -0
- gaphack/core.py +1216 -0
- gaphack/distance_providers.py +196 -0
- gaphack/neighborhood_finder.py +55 -0
- gaphack/refine_cli.py +832 -0
- gaphack/refinement_types.py +143 -0
- gaphack/target_clustering.py +384 -0
- gaphack/utils.py +909 -0
- gaphack/version.py +6 -0
- gaphack/vsearch_neighborhood.py +325 -0
- gaphack-0.7.1.dist-info/METADATA +902 -0
- gaphack-0.7.1.dist-info/RECORD +24 -0
- gaphack-0.7.1.dist-info/WHEEL +5 -0
- gaphack-0.7.1.dist-info/entry_points.txt +5 -0
- gaphack-0.7.1.dist-info/licenses/LICENSE +24 -0
- gaphack-0.7.1.dist-info/top_level.txt +1 -0
gaphack/__init__.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""
|
|
2
|
+
gapHACk: Gap-Optimized Hierarchical Agglomerative Clustering
|
|
3
|
+
|
|
4
|
+
A Python package for DNA barcode clustering that optimizes for the barcode gap
|
|
5
|
+
between intra-species and inter-species genetic distances.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "0.7.0"
|
|
9
|
+
|
|
10
|
+
from .core import GapOptimizedClustering
|
|
11
|
+
from .target_clustering import TargetModeClustering
|
|
12
|
+
from .distance_providers import (
|
|
13
|
+
DistanceProvider,
|
|
14
|
+
MSACachedDistanceProvider,
|
|
15
|
+
PrecomputedDistanceProvider,
|
|
16
|
+
MSAAlignmentError
|
|
17
|
+
)
|
|
18
|
+
from .utils import (
|
|
19
|
+
calculate_distance_matrix,
|
|
20
|
+
load_sequences_from_fasta,
|
|
21
|
+
save_clusters_to_file,
|
|
22
|
+
format_cluster_output,
|
|
23
|
+
validate_sequences
|
|
24
|
+
)
|
|
25
|
+
from .analyze import (
|
|
26
|
+
calculate_intra_cluster_distances,
|
|
27
|
+
calculate_inter_cluster_distances,
|
|
28
|
+
calculate_percentiles,
|
|
29
|
+
calculate_barcode_gap_metrics,
|
|
30
|
+
create_histogram,
|
|
31
|
+
create_combined_histogram,
|
|
32
|
+
format_analysis_report
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"GapOptimizedClustering",
|
|
37
|
+
"TargetModeClustering",
|
|
38
|
+
"DistanceProvider",
|
|
39
|
+
"MSACachedDistanceProvider",
|
|
40
|
+
"PrecomputedDistanceProvider",
|
|
41
|
+
"MSAAlignmentError",
|
|
42
|
+
"calculate_distance_matrix",
|
|
43
|
+
"load_sequences_from_fasta",
|
|
44
|
+
"save_clusters_to_file",
|
|
45
|
+
"format_cluster_output",
|
|
46
|
+
"validate_sequences",
|
|
47
|
+
"calculate_intra_cluster_distances",
|
|
48
|
+
"calculate_inter_cluster_distances",
|
|
49
|
+
"calculate_percentiles",
|
|
50
|
+
"calculate_barcode_gap_metrics",
|
|
51
|
+
"create_histogram",
|
|
52
|
+
"create_combined_histogram",
|
|
53
|
+
"format_analysis_report"
|
|
54
|
+
]
|
gaphack/analyze.py
ADDED
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Analysis functions for pre-clustered FASTA files.
|
|
3
|
+
|
|
4
|
+
This module provides functionality to analyze distance distributions and barcode gaps
|
|
5
|
+
from FASTA files that are assumed to contain pre-clustered sequences.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import matplotlib.pyplot as plt
|
|
10
|
+
from typing import List, Dict, Tuple, Optional
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import logging
|
|
13
|
+
from .utils import load_sequences_from_fasta, calculate_distance_matrix
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def calculate_intra_cluster_distances(sequences: List[str]) -> np.ndarray:
|
|
17
|
+
"""
|
|
18
|
+
Calculate all pairwise distances within a cluster.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
sequences: List of DNA sequences
|
|
22
|
+
|
|
23
|
+
Returns:
|
|
24
|
+
1D array of pairwise distances (upper triangle, excluding diagonal)
|
|
25
|
+
"""
|
|
26
|
+
if len(sequences) < 2:
|
|
27
|
+
return np.array([])
|
|
28
|
+
|
|
29
|
+
distance_matrix = calculate_distance_matrix(sequences)
|
|
30
|
+
|
|
31
|
+
# Extract upper triangle (excluding diagonal)
|
|
32
|
+
n = len(sequences)
|
|
33
|
+
distances = []
|
|
34
|
+
for i in range(n):
|
|
35
|
+
for j in range(i + 1, n):
|
|
36
|
+
distances.append(distance_matrix[i, j])
|
|
37
|
+
|
|
38
|
+
return np.array(distances)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def calculate_inter_cluster_distances(cluster_sequences: List[List[str]]) -> np.ndarray:
|
|
42
|
+
"""
|
|
43
|
+
Calculate all pairwise distances between different clusters.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
cluster_sequences: List of sequence lists, one per cluster
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
1D array of inter-cluster pairwise distances
|
|
50
|
+
"""
|
|
51
|
+
if len(cluster_sequences) < 2:
|
|
52
|
+
return np.array([])
|
|
53
|
+
|
|
54
|
+
# Flatten all sequences and track cluster membership
|
|
55
|
+
all_sequences = []
|
|
56
|
+
cluster_membership = []
|
|
57
|
+
|
|
58
|
+
for cluster_idx, sequences in enumerate(cluster_sequences):
|
|
59
|
+
all_sequences.extend(sequences)
|
|
60
|
+
cluster_membership.extend([cluster_idx] * len(sequences))
|
|
61
|
+
|
|
62
|
+
if len(all_sequences) < 2:
|
|
63
|
+
return np.array([])
|
|
64
|
+
|
|
65
|
+
# Calculate full distance matrix
|
|
66
|
+
distance_matrix = calculate_distance_matrix(all_sequences)
|
|
67
|
+
|
|
68
|
+
# Extract inter-cluster distances
|
|
69
|
+
inter_distances = []
|
|
70
|
+
n = len(all_sequences)
|
|
71
|
+
|
|
72
|
+
for i in range(n):
|
|
73
|
+
for j in range(i + 1, n):
|
|
74
|
+
if cluster_membership[i] != cluster_membership[j]:
|
|
75
|
+
inter_distances.append(distance_matrix[i, j])
|
|
76
|
+
|
|
77
|
+
return np.array(inter_distances)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def calculate_percentiles(distances: np.ndarray,
|
|
81
|
+
percentiles: List[float] = [5, 10, 25, 50, 75, 90, 95, 99, 100]) -> Dict[str, float]:
|
|
82
|
+
"""
|
|
83
|
+
Calculate key percentile values for a set of distances.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
distances: Array of distance values
|
|
87
|
+
percentiles: List of percentiles to calculate
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
Dictionary mapping percentile names to values
|
|
91
|
+
"""
|
|
92
|
+
# Filter out NaN values
|
|
93
|
+
distances = distances[~np.isnan(distances)] if len(distances) > 0 else distances
|
|
94
|
+
|
|
95
|
+
if len(distances) == 0:
|
|
96
|
+
return {f"P{p}": np.nan for p in percentiles}
|
|
97
|
+
|
|
98
|
+
percentile_values = np.percentile(distances, percentiles)
|
|
99
|
+
return {f"P{int(p)}": float(val) for p, val in zip(percentiles, percentile_values)}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def calculate_barcode_gap_metrics(intra_distances: np.ndarray,
|
|
103
|
+
inter_distances: np.ndarray,
|
|
104
|
+
target_percentiles: List[int] = [90, 95]) -> Dict[str, Dict[str, float]]:
|
|
105
|
+
"""
|
|
106
|
+
Calculate barcode gap metrics for given intra and inter-cluster distances.
|
|
107
|
+
|
|
108
|
+
Args:
|
|
109
|
+
intra_distances: Array of intra-cluster distances
|
|
110
|
+
inter_distances: Array of inter-cluster distances
|
|
111
|
+
target_percentiles: Percentiles to calculate gaps for
|
|
112
|
+
|
|
113
|
+
Returns:
|
|
114
|
+
Dictionary with gap metrics for each percentile
|
|
115
|
+
"""
|
|
116
|
+
gap_metrics = {}
|
|
117
|
+
|
|
118
|
+
# Filter out NaN values
|
|
119
|
+
intra_distances = intra_distances[~np.isnan(intra_distances)] if len(intra_distances) > 0 else intra_distances
|
|
120
|
+
inter_distances = inter_distances[~np.isnan(inter_distances)] if len(inter_distances) > 0 else inter_distances
|
|
121
|
+
|
|
122
|
+
if len(intra_distances) == 0 or len(inter_distances) == 0:
|
|
123
|
+
for percentile in target_percentiles:
|
|
124
|
+
gap_metrics[f"P{percentile}"] = {
|
|
125
|
+
"intra_percentile": np.nan,
|
|
126
|
+
"inter_percentile": np.nan,
|
|
127
|
+
"gap_size": np.nan,
|
|
128
|
+
"gap_exists": False
|
|
129
|
+
}
|
|
130
|
+
return gap_metrics
|
|
131
|
+
|
|
132
|
+
for percentile in target_percentiles:
|
|
133
|
+
intra_threshold = np.percentile(intra_distances, percentile)
|
|
134
|
+
inter_threshold = np.percentile(inter_distances, 100 - percentile)
|
|
135
|
+
|
|
136
|
+
gap_size = inter_threshold - intra_threshold
|
|
137
|
+
gap_exists = gap_size > 0
|
|
138
|
+
|
|
139
|
+
gap_metrics[f"P{percentile}"] = {
|
|
140
|
+
"intra_percentile": float(intra_threshold),
|
|
141
|
+
"inter_percentile": float(inter_threshold),
|
|
142
|
+
"gap_size": float(gap_size),
|
|
143
|
+
"gap_exists": gap_exists
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return gap_metrics
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def create_histogram(distances: np.ndarray,
|
|
150
|
+
title: str,
|
|
151
|
+
xlabel: str = "Distance",
|
|
152
|
+
bins: int = 50,
|
|
153
|
+
save_path: Optional[str] = None,
|
|
154
|
+
show_percentiles: bool = True) -> plt.Figure:
|
|
155
|
+
"""
|
|
156
|
+
Create a histogram of distances with optional percentile markers.
|
|
157
|
+
|
|
158
|
+
Args:
|
|
159
|
+
distances: Array of distance values
|
|
160
|
+
title: Title for the histogram
|
|
161
|
+
xlabel: Label for x-axis
|
|
162
|
+
bins: Number of histogram bins
|
|
163
|
+
save_path: Optional path to save the figure
|
|
164
|
+
show_percentiles: Whether to show percentile markers
|
|
165
|
+
|
|
166
|
+
Returns:
|
|
167
|
+
matplotlib Figure object
|
|
168
|
+
"""
|
|
169
|
+
fig, ax = plt.subplots(figsize=(10, 6))
|
|
170
|
+
|
|
171
|
+
# Ensure distances is a numpy array
|
|
172
|
+
distances = np.asarray(distances)
|
|
173
|
+
|
|
174
|
+
# Filter out NaN values
|
|
175
|
+
distances = distances[~np.isnan(distances)] if len(distances) > 0 else distances
|
|
176
|
+
|
|
177
|
+
if len(distances) == 0:
|
|
178
|
+
ax.text(0.5, 0.5, 'No distances available',
|
|
179
|
+
ha='center', va='center', transform=ax.transAxes)
|
|
180
|
+
ax.set_title(title)
|
|
181
|
+
ax.set_xlabel(xlabel)
|
|
182
|
+
ax.set_ylabel("Frequency")
|
|
183
|
+
return fig
|
|
184
|
+
|
|
185
|
+
# Create histogram
|
|
186
|
+
counts, bin_edges, patches = ax.hist(distances, bins=bins, alpha=0.7, edgecolor='black')
|
|
187
|
+
|
|
188
|
+
# Add percentile markers if requested
|
|
189
|
+
if show_percentiles:
|
|
190
|
+
percentiles_to_show = [5, 25, 50, 75, 95]
|
|
191
|
+
percentile_values = np.percentile(distances, percentiles_to_show)
|
|
192
|
+
colors = ['red', 'orange', 'green', 'orange', 'red']
|
|
193
|
+
|
|
194
|
+
for p, val, color in zip(percentiles_to_show, percentile_values, colors):
|
|
195
|
+
ax.axvline(val, color=color, linestyle='--', alpha=0.8,
|
|
196
|
+
label=f'P{p}: {val:.4f}')
|
|
197
|
+
|
|
198
|
+
ax.set_title(title)
|
|
199
|
+
ax.set_xlabel(xlabel)
|
|
200
|
+
ax.set_ylabel("Frequency")
|
|
201
|
+
ax.grid(True, alpha=0.3)
|
|
202
|
+
|
|
203
|
+
if show_percentiles:
|
|
204
|
+
ax.legend()
|
|
205
|
+
|
|
206
|
+
# Add statistics text
|
|
207
|
+
if len(distances) > 0:
|
|
208
|
+
stats_text = f"n={len(distances):,}\nMean: {np.mean(distances):.4f}\nStd: {np.std(distances):.4f}"
|
|
209
|
+
ax.text(0.02, 0.98, stats_text, transform=ax.transAxes,
|
|
210
|
+
verticalalignment='top', bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.8))
|
|
211
|
+
|
|
212
|
+
plt.tight_layout()
|
|
213
|
+
|
|
214
|
+
if save_path:
|
|
215
|
+
fig.savefig(save_path, dpi=300, bbox_inches='tight')
|
|
216
|
+
logging.info(f"Histogram saved to {save_path}")
|
|
217
|
+
|
|
218
|
+
return fig
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def create_combined_histogram(intra_distances: np.ndarray,
|
|
222
|
+
inter_distances: np.ndarray,
|
|
223
|
+
title: str = "Global Distance Distribution",
|
|
224
|
+
bins: int = 50,
|
|
225
|
+
save_path: Optional[str] = None) -> plt.Figure:
|
|
226
|
+
"""
|
|
227
|
+
Create a combined histogram showing both intra and inter-cluster distances.
|
|
228
|
+
|
|
229
|
+
Args:
|
|
230
|
+
intra_distances: Array of intra-cluster distances
|
|
231
|
+
inter_distances: Array of inter-cluster distances
|
|
232
|
+
title: Title for the histogram
|
|
233
|
+
bins: Number of histogram bins
|
|
234
|
+
save_path: Optional path to save the figure
|
|
235
|
+
|
|
236
|
+
Returns:
|
|
237
|
+
matplotlib Figure object
|
|
238
|
+
"""
|
|
239
|
+
fig, ax = plt.subplots(figsize=(12, 8))
|
|
240
|
+
|
|
241
|
+
# Determine common bin range (filter out NaN values)
|
|
242
|
+
all_distances = np.concatenate([intra_distances, inter_distances]) if len(intra_distances) > 0 and len(inter_distances) > 0 else np.array([])
|
|
243
|
+
all_distances = all_distances[~np.isnan(all_distances)]
|
|
244
|
+
|
|
245
|
+
if len(all_distances) == 0:
|
|
246
|
+
ax.text(0.5, 0.5, 'No distances available',
|
|
247
|
+
ha='center', va='center', transform=ax.transAxes)
|
|
248
|
+
ax.set_title(title)
|
|
249
|
+
return fig
|
|
250
|
+
|
|
251
|
+
bin_range = (all_distances.min(), all_distances.max())
|
|
252
|
+
|
|
253
|
+
# Filter NaN from individual arrays for histograms
|
|
254
|
+
intra_distances = intra_distances[~np.isnan(intra_distances)] if len(intra_distances) > 0 else intra_distances
|
|
255
|
+
inter_distances = inter_distances[~np.isnan(inter_distances)] if len(inter_distances) > 0 else inter_distances
|
|
256
|
+
|
|
257
|
+
# Create histograms
|
|
258
|
+
if len(intra_distances) > 0:
|
|
259
|
+
ax.hist(intra_distances, bins=bins, alpha=0.6, label=f'Intra-cluster (n={len(intra_distances):,})',
|
|
260
|
+
color='blue', range=bin_range)
|
|
261
|
+
|
|
262
|
+
if len(inter_distances) > 0:
|
|
263
|
+
ax.hist(inter_distances, bins=bins, alpha=0.6, label=f'Inter-cluster (n={len(inter_distances):,})',
|
|
264
|
+
color='red', range=bin_range)
|
|
265
|
+
|
|
266
|
+
ax.set_title(title)
|
|
267
|
+
ax.set_xlabel("Distance")
|
|
268
|
+
ax.set_ylabel("Frequency")
|
|
269
|
+
ax.grid(True, alpha=0.3)
|
|
270
|
+
ax.legend()
|
|
271
|
+
|
|
272
|
+
plt.tight_layout()
|
|
273
|
+
|
|
274
|
+
if save_path:
|
|
275
|
+
fig.savefig(save_path, dpi=300, bbox_inches='tight')
|
|
276
|
+
logging.info(f"Combined histogram saved to {save_path}")
|
|
277
|
+
|
|
278
|
+
return fig
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def format_analysis_report(cluster_stats: List[Dict],
|
|
282
|
+
global_stats: Dict,
|
|
283
|
+
gap_metrics: Dict) -> str:
|
|
284
|
+
"""
|
|
285
|
+
Format analysis results into a readable text report.
|
|
286
|
+
|
|
287
|
+
Args:
|
|
288
|
+
cluster_stats: List of statistics for each individual cluster
|
|
289
|
+
global_stats: Global statistics dictionary
|
|
290
|
+
gap_metrics: Barcode gap metrics
|
|
291
|
+
|
|
292
|
+
Returns:
|
|
293
|
+
Formatted report string
|
|
294
|
+
"""
|
|
295
|
+
lines = []
|
|
296
|
+
lines.append("=" * 80)
|
|
297
|
+
lines.append("GAPHACK CLUSTER ANALYSIS REPORT")
|
|
298
|
+
lines.append("=" * 80)
|
|
299
|
+
lines.append("")
|
|
300
|
+
|
|
301
|
+
# Individual cluster analysis
|
|
302
|
+
lines.append("INDIVIDUAL CLUSTER ANALYSIS")
|
|
303
|
+
lines.append("-" * 40)
|
|
304
|
+
|
|
305
|
+
for i, stats in enumerate(cluster_stats):
|
|
306
|
+
lines.append(f"\nCluster {i+1}: {stats['filename']}")
|
|
307
|
+
lines.append(f" Sequences: {stats['n_sequences']:,}")
|
|
308
|
+
lines.append(f" Distances: {stats['n_distances']:,}")
|
|
309
|
+
|
|
310
|
+
if stats['n_distances'] > 0:
|
|
311
|
+
lines.append(" Percentiles:")
|
|
312
|
+
for key, value in stats['percentiles'].items():
|
|
313
|
+
lines.append(f" {key}: {value:.6f}")
|
|
314
|
+
else:
|
|
315
|
+
lines.append(" No pairwise distances (single sequence)")
|
|
316
|
+
lines.append("")
|
|
317
|
+
|
|
318
|
+
# Global analysis
|
|
319
|
+
lines.append("GLOBAL ANALYSIS")
|
|
320
|
+
lines.append("-" * 40)
|
|
321
|
+
lines.append(f"Total clusters analyzed: {len(cluster_stats)}")
|
|
322
|
+
lines.append(f"Total sequences: {global_stats.get('total_sequences', 0):,}")
|
|
323
|
+
lines.append(f"Total intra-cluster distances: {global_stats.get('n_intra', 0):,}")
|
|
324
|
+
lines.append(f"Total inter-cluster distances: {global_stats.get('n_inter', 0):,}")
|
|
325
|
+
lines.append("")
|
|
326
|
+
|
|
327
|
+
if global_stats.get('n_intra', 0) > 0:
|
|
328
|
+
lines.append("Intra-cluster distance percentiles:")
|
|
329
|
+
for key, value in global_stats.get('intra_percentiles', {}).items():
|
|
330
|
+
lines.append(f" {key}: {value:.6f}")
|
|
331
|
+
lines.append("")
|
|
332
|
+
|
|
333
|
+
if global_stats.get('n_inter', 0) > 0:
|
|
334
|
+
lines.append("Inter-cluster distance percentiles:")
|
|
335
|
+
for key, value in global_stats.get('inter_percentiles', {}).items():
|
|
336
|
+
lines.append(f" {key}: {value:.6f}")
|
|
337
|
+
lines.append("")
|
|
338
|
+
|
|
339
|
+
# Barcode gap analysis
|
|
340
|
+
lines.append("BARCODE GAP ANALYSIS")
|
|
341
|
+
lines.append("-" * 40)
|
|
342
|
+
|
|
343
|
+
if len(gap_metrics) > 0:
|
|
344
|
+
for percentile, metrics in gap_metrics.items():
|
|
345
|
+
lines.append(f"\n{percentile} Gap Analysis:")
|
|
346
|
+
lines.append(f" Intra-cluster {percentile}: {metrics['intra_percentile']:.6f}")
|
|
347
|
+
lines.append(f" Inter-cluster {100-int(percentile[1:])}th percentile: {metrics['inter_percentile']:.6f}")
|
|
348
|
+
lines.append(f" Gap size: {metrics['gap_size']:.6f}")
|
|
349
|
+
lines.append(f" Gap exists: {'Yes' if metrics['gap_exists'] else 'No'}")
|
|
350
|
+
else:
|
|
351
|
+
lines.append("Cannot calculate barcode gap (insufficient data)")
|
|
352
|
+
|
|
353
|
+
lines.append("")
|
|
354
|
+
lines.append("=" * 80)
|
|
355
|
+
|
|
356
|
+
return "\n".join(lines)
|