lattice-sub 1.0.10__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/MANIFEST.in +12 -0
- {lattice_sub-1.0.10/src/lattice_sub.egg-info → lattice_sub-1.1.0}/PKG-INFO +90 -20
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/README.md +88 -19
- lattice_sub-1.1.0/docs/images/example_comparison.png +0 -0
- lattice_sub-1.1.0/docs/images/threshold_analysis.png +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/examples/config.yaml +17 -3
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/pyproject.toml +2 -1
- {lattice_sub-1.0.10 → lattice_sub-1.1.0/src/lattice_sub.egg-info}/PKG-INFO +90 -20
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_sub.egg-info/SOURCES.txt +3 -3
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_sub.egg-info/requires.txt +1 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/__init__.py +11 -1
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/cli.py +12 -8
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/config.py +17 -4
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/core.py +31 -5
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/processing.py +62 -0
- lattice_sub-1.1.0/src/lattice_subtraction/threshold_optimizer.py +436 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/visualization.py +4 -0
- lattice_sub-1.0.10/docs/architecture.md +0 -245
- lattice_sub-1.0.10/docs/images/example_comparison.png +0 -0
- lattice_sub-1.0.10/tests/test_lattice_subtraction.py +0 -189
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/LICENSE +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/examples/converted_params.yaml +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/setup.cfg +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_sub.egg-info/dependency_links.txt +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_sub.egg-info/entry_points.txt +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_sub.egg-info/top_level.txt +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/batch.py +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/io.py +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/masks.py +0 -0
- {lattice_sub-1.0.10 → lattice_sub-1.1.0}/src/lattice_subtraction/ui.py +0 -0
|
@@ -10,10 +10,22 @@ recursive-include src *.py
|
|
|
10
10
|
recursive-include examples *.yaml *.yml
|
|
11
11
|
recursive-include docs *.md *.png
|
|
12
12
|
|
|
13
|
+
# Exclude tests (dev only - available on GitHub)
|
|
14
|
+
prune tests
|
|
15
|
+
|
|
13
16
|
# Exclude internal/development files
|
|
14
17
|
exclude NOTES_*.md
|
|
15
18
|
exclude *.log
|
|
19
|
+
exclude benchmark_*.py
|
|
20
|
+
exclude analyze_thresholds.py
|
|
21
|
+
exclude test_search_strategies.py
|
|
22
|
+
prune benchmark_visualizations
|
|
23
|
+
prune kornia_benchmark_visualizations
|
|
24
|
+
prune dist
|
|
25
|
+
prune tests/test_output
|
|
26
|
+
prune tests/test_output_gpu
|
|
16
27
|
global-exclude *.pyc
|
|
17
28
|
global-exclude __pycache__
|
|
18
29
|
global-exclude *.egg-info
|
|
19
30
|
global-exclude .git*
|
|
31
|
+
global-exclude .pytest_cache
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lattice-sub
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Lattice subtraction for cryo-EM micrographs - removes periodic crystal signals to reveal non-periodic features
|
|
5
5
|
Author-email: George Stephenson <george.stephenson@colorado.edu>, Vignesh Kasinath <vignesh.kasinath@colorado.edu>
|
|
6
6
|
License: MIT
|
|
@@ -28,6 +28,7 @@ Requires-Dist: click>=8.1
|
|
|
28
28
|
Requires-Dist: scikit-image>=0.21
|
|
29
29
|
Requires-Dist: torch>=2.0
|
|
30
30
|
Requires-Dist: matplotlib>=3.7
|
|
31
|
+
Requires-Dist: kornia>=0.7
|
|
31
32
|
Provides-Extra: dev
|
|
32
33
|
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
33
34
|
Requires-Dist: pytest-cov; extra == "dev"
|
|
@@ -67,6 +68,8 @@ pip install lattice-sub
|
|
|
67
68
|
|
|
68
69
|
That's it! GPU acceleration works automatically if you have an NVIDIA GPU.
|
|
69
70
|
|
|
71
|
+
> **Note:** Requires Python 3.11+ and an NVIDIA GPU (RTX 20/30/40 series, A100, etc.) for best performance. CPU fallback is available but slower.
|
|
72
|
+
|
|
70
73
|
---
|
|
71
74
|
|
|
72
75
|
## Quick Start
|
|
@@ -77,6 +80,8 @@ That's it! GPU acceleration works automatically if you have an NVIDIA GPU.
|
|
|
77
80
|
lattice-sub process your_image.mrc -o output.mrc --pixel-size 0.56
|
|
78
81
|
```
|
|
79
82
|
|
|
83
|
+
> **Pixel size:** Use your detector's actual pixel size (e.g., K3=0.56Å, Falcon=1.14Å, K2=1.32Å)
|
|
84
|
+
|
|
80
85
|
### Process a Folder of Images
|
|
81
86
|
|
|
82
87
|
```bash
|
|
@@ -99,7 +104,7 @@ This creates side-by-side PNG images showing before/after/difference for each mi
|
|
|
99
104
|
|--------|-------------|
|
|
100
105
|
| `-p, --pixel-size` | **Required.** Pixel size in Ångstroms |
|
|
101
106
|
| `-o, --output` | Output file path (default: `sub_<input>`) |
|
|
102
|
-
| `-t, --threshold` | Peak detection sensitivity (default:
|
|
107
|
+
| `-t, --threshold` | Peak detection sensitivity (default: **auto** - optimized per image) |
|
|
103
108
|
| `--cpu` | Force CPU processing (GPU is used by default) |
|
|
104
109
|
| `-q, --quiet` | Hide the banner and progress messages |
|
|
105
110
|
| `-v, --verbose` | Show detailed processing information |
|
|
@@ -107,7 +112,10 @@ This creates side-by-side PNG images showing before/after/difference for each mi
|
|
|
107
112
|
### Example with Options
|
|
108
113
|
|
|
109
114
|
```bash
|
|
110
|
-
# Process with
|
|
115
|
+
# Process with auto-optimized threshold (default - recommended)
|
|
116
|
+
lattice-sub process image.mrc -o cleaned.mrc -p 0.56 -v
|
|
117
|
+
|
|
118
|
+
# Override with a specific threshold if needed
|
|
111
119
|
lattice-sub process image.mrc -o cleaned.mrc -p 0.56 -t 1.5 -v
|
|
112
120
|
|
|
113
121
|
# Batch process, force CPU with 8 parallel workers
|
|
@@ -116,16 +124,22 @@ lattice-sub batch raw/ processed/ -p 0.56 --cpu -j 8
|
|
|
116
124
|
|
|
117
125
|
---
|
|
118
126
|
|
|
119
|
-
## Using a Config File
|
|
127
|
+
## Using a Config File (Optional)
|
|
128
|
+
|
|
129
|
+
For most users, the defaults work great — just use `-p` for pixel size:
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
lattice-sub batch input/ output/ -p 0.56
|
|
133
|
+
```
|
|
120
134
|
|
|
121
|
-
|
|
135
|
+
Config files are useful for **reproducibility** or **non-standard samples**:
|
|
122
136
|
|
|
123
137
|
```yaml
|
|
124
|
-
# params.yaml
|
|
138
|
+
# params.yaml - only include what you want to override
|
|
125
139
|
pixel_ang: 0.56
|
|
126
|
-
|
|
127
|
-
inside_radius_ang: 90
|
|
128
|
-
|
|
140
|
+
unit_cell_ang: 120 # Default: 116 (nucleosome). Change for other crystals.
|
|
141
|
+
inside_radius_ang: 80 # Default: 90. Protect different resolution range.
|
|
142
|
+
# threshold: 1.45 # Uncomment to override auto-optimization
|
|
129
143
|
```
|
|
130
144
|
|
|
131
145
|
Then use it:
|
|
@@ -244,6 +258,60 @@ MIT License - see [LICENSE](LICENSE) for details.
|
|
|
244
258
|
<details>
|
|
245
259
|
<summary><strong>📚 Advanced Topics</strong> (click to expand)</summary>
|
|
246
260
|
|
|
261
|
+
### v1.1.0 Optimizations
|
|
262
|
+
|
|
263
|
+
Version 1.1.0 introduces two major optimizations that make the tool both **faster** and **smarter**:
|
|
264
|
+
|
|
265
|
+
#### 1. Adaptive Per-Image Threshold Optimization
|
|
266
|
+
|
|
267
|
+
**Problem (v1.0.x):** A fixed threshold (1.42) was used for all images, but optimal thresholds vary by image quality, ice thickness, and lattice order.
|
|
268
|
+
|
|
269
|
+
**Solution (v1.1.0):** Automatic per-image optimization using grid search:
|
|
270
|
+
- Tests 21 threshold values in range [1.40, 1.60] with 0.01 step
|
|
271
|
+
- Scores each using a quality function that balances:
|
|
272
|
+
- **Peak count** (target: ~600 peaks for good lattice coverage)
|
|
273
|
+
- **Peak SNR** (signal-to-noise of detected peaks)
|
|
274
|
+
- **Peak distribution** (uniform hexagonal spacing preferred)
|
|
275
|
+
- **Coverage** (adequate sampling across frequency space)
|
|
276
|
+
- Returns the threshold with highest quality score
|
|
277
|
+
|
|
278
|
+
**Result:** Each image gets its optimal threshold automatically.
|
|
279
|
+
|
|
280
|
+

|
|
281
|
+
|
|
282
|
+
#### 2. GPU-Accelerated Background Subtraction (Kornia)
|
|
283
|
+
|
|
284
|
+
**Problem (v1.0.x):** Profiling revealed background subtraction (scipy median filter) consumed **94% of processing time** — not FFT as expected.
|
|
285
|
+
|
|
286
|
+
**Solution (v1.1.0):** Replaced scipy's CPU median filter with Kornia's GPU implementation:
|
|
287
|
+
```python
|
|
288
|
+
# Before (v1.0.x) - CPU, ~2.5s per image
|
|
289
|
+
from scipy.ndimage import median_filter
|
|
290
|
+
background = median_filter(log_power, size=51)
|
|
291
|
+
|
|
292
|
+
# After (v1.1.0) - GPU, ~0.05s per image
|
|
293
|
+
from kornia.filters import median_blur
|
|
294
|
+
background = median_blur(log_power_tensor, (51, 51))
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
**Result:** 48x speedup on background subtraction alone.
|
|
298
|
+
|
|
299
|
+
#### Performance Comparison
|
|
300
|
+
|
|
301
|
+
| Version | Threshold | Background Sub | Time/Image | Quality |
|
|
302
|
+
|---------|-----------|----------------|------------|---------|
|
|
303
|
+
| v1.0.10 | Fixed (1.42) | scipy CPU | ~12s | Good |
|
|
304
|
+
| v1.1.0 | Fixed | Kornia GPU | ~1.0s | Good |
|
|
305
|
+
| v1.1.0 | **Auto** | **Kornia GPU** | **~2.6s** | **Optimal** |
|
|
306
|
+
|
|
307
|
+
**Net result:** 5x faster with better results per image.
|
|
308
|
+
|
|
309
|
+
#### Correlation Validation
|
|
310
|
+
|
|
311
|
+
Kornia GPU vs scipy CPU background subtraction correlation: **0.9976** (nearly identical output).
|
|
312
|
+
|
|
313
|
+
---
|
|
314
|
+
|
|
247
315
|
### Algorithm Details
|
|
248
316
|
|
|
249
317
|
```
|
|
@@ -264,12 +332,13 @@ The algorithm:
|
|
|
264
332
|
| Parameter | Default | Description |
|
|
265
333
|
|-----------|---------|-------------|
|
|
266
334
|
| `pixel_ang` | *required* | Pixel size in Ångstroms |
|
|
267
|
-
| `threshold` |
|
|
335
|
+
| `threshold` | **auto** | Peak detection threshold - auto-optimized per image (range 1.40-1.60) |
|
|
268
336
|
| `inside_radius_ang` | 90 | Inner resolution limit (Å) - protects structural info |
|
|
269
337
|
| `outside_radius_ang` | auto | Outer resolution limit (Å) - protects near Nyquist |
|
|
270
338
|
| `expand_pixel` | 10 | Morphological expansion of peak mask (pixels) |
|
|
271
339
|
| `unit_cell_ang` | 116 | Crystal unit cell for inpaint shift calculation (Å) |
|
|
272
340
|
| `backend` | auto | `"auto"`, `"numpy"` (CPU), or `"pytorch"` (GPU) |
|
|
341
|
+
| `use_kornia` | **true** | Use Kornia for GPU-accelerated background subtraction (48x faster) |
|
|
273
342
|
|
|
274
343
|
### Supported Hardware
|
|
275
344
|
|
|
@@ -294,16 +363,17 @@ pytest tests/ -v # Run tests
|
|
|
294
363
|
|
|
295
364
|
```
|
|
296
365
|
src/lattice_subtraction/
|
|
297
|
-
├── __init__.py
|
|
298
|
-
├── cli.py
|
|
299
|
-
├── core.py
|
|
300
|
-
├── batch.py
|
|
301
|
-
├── config.py
|
|
302
|
-
├── io.py
|
|
303
|
-
├── masks.py
|
|
304
|
-
├── processing.py
|
|
305
|
-
├──
|
|
306
|
-
|
|
366
|
+
├── __init__.py # Package exports
|
|
367
|
+
├── cli.py # Command-line interface
|
|
368
|
+
├── core.py # LatticeSubtractor main class
|
|
369
|
+
├── batch.py # Parallel batch processing
|
|
370
|
+
├── config.py # Configuration dataclass
|
|
371
|
+
├── io.py # MRC file I/O
|
|
372
|
+
├── masks.py # FFT mask generation
|
|
373
|
+
├── processing.py # FFT helpers + GPU background subtraction
|
|
374
|
+
├── threshold_optimizer.py # Auto-threshold optimization (NEW in v1.1)
|
|
375
|
+
├── ui.py # Terminal UI
|
|
376
|
+
└── visualization.py # Comparison figures
|
|
307
377
|
```
|
|
308
378
|
|
|
309
379
|
### Migration from MATLAB
|
|
@@ -27,6 +27,8 @@ pip install lattice-sub
|
|
|
27
27
|
|
|
28
28
|
That's it! GPU acceleration works automatically if you have an NVIDIA GPU.
|
|
29
29
|
|
|
30
|
+
> **Note:** Requires Python 3.11+ and an NVIDIA GPU (RTX 20/30/40 series, A100, etc.) for best performance. CPU fallback is available but slower.
|
|
31
|
+
|
|
30
32
|
---
|
|
31
33
|
|
|
32
34
|
## Quick Start
|
|
@@ -37,6 +39,8 @@ That's it! GPU acceleration works automatically if you have an NVIDIA GPU.
|
|
|
37
39
|
lattice-sub process your_image.mrc -o output.mrc --pixel-size 0.56
|
|
38
40
|
```
|
|
39
41
|
|
|
42
|
+
> **Pixel size:** Use your detector's actual pixel size (e.g., K3=0.56Å, Falcon=1.14Å, K2=1.32Å)
|
|
43
|
+
|
|
40
44
|
### Process a Folder of Images
|
|
41
45
|
|
|
42
46
|
```bash
|
|
@@ -59,7 +63,7 @@ This creates side-by-side PNG images showing before/after/difference for each mi
|
|
|
59
63
|
|--------|-------------|
|
|
60
64
|
| `-p, --pixel-size` | **Required.** Pixel size in Ångstroms |
|
|
61
65
|
| `-o, --output` | Output file path (default: `sub_<input>`) |
|
|
62
|
-
| `-t, --threshold` | Peak detection sensitivity (default:
|
|
66
|
+
| `-t, --threshold` | Peak detection sensitivity (default: **auto** - optimized per image) |
|
|
63
67
|
| `--cpu` | Force CPU processing (GPU is used by default) |
|
|
64
68
|
| `-q, --quiet` | Hide the banner and progress messages |
|
|
65
69
|
| `-v, --verbose` | Show detailed processing information |
|
|
@@ -67,7 +71,10 @@ This creates side-by-side PNG images showing before/after/difference for each mi
|
|
|
67
71
|
### Example with Options
|
|
68
72
|
|
|
69
73
|
```bash
|
|
70
|
-
# Process with
|
|
74
|
+
# Process with auto-optimized threshold (default - recommended)
|
|
75
|
+
lattice-sub process image.mrc -o cleaned.mrc -p 0.56 -v
|
|
76
|
+
|
|
77
|
+
# Override with a specific threshold if needed
|
|
71
78
|
lattice-sub process image.mrc -o cleaned.mrc -p 0.56 -t 1.5 -v
|
|
72
79
|
|
|
73
80
|
# Batch process, force CPU with 8 parallel workers
|
|
@@ -76,16 +83,22 @@ lattice-sub batch raw/ processed/ -p 0.56 --cpu -j 8
|
|
|
76
83
|
|
|
77
84
|
---
|
|
78
85
|
|
|
79
|
-
## Using a Config File
|
|
86
|
+
## Using a Config File (Optional)
|
|
87
|
+
|
|
88
|
+
For most users, the defaults work great — just use `-p` for pixel size:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
lattice-sub batch input/ output/ -p 0.56
|
|
92
|
+
```
|
|
80
93
|
|
|
81
|
-
|
|
94
|
+
Config files are useful for **reproducibility** or **non-standard samples**:
|
|
82
95
|
|
|
83
96
|
```yaml
|
|
84
|
-
# params.yaml
|
|
97
|
+
# params.yaml - only include what you want to override
|
|
85
98
|
pixel_ang: 0.56
|
|
86
|
-
|
|
87
|
-
inside_radius_ang: 90
|
|
88
|
-
|
|
99
|
+
unit_cell_ang: 120 # Default: 116 (nucleosome). Change for other crystals.
|
|
100
|
+
inside_radius_ang: 80 # Default: 90. Protect different resolution range.
|
|
101
|
+
# threshold: 1.45 # Uncomment to override auto-optimization
|
|
89
102
|
```
|
|
90
103
|
|
|
91
104
|
Then use it:
|
|
@@ -204,6 +217,60 @@ MIT License - see [LICENSE](LICENSE) for details.
|
|
|
204
217
|
<details>
|
|
205
218
|
<summary><strong>📚 Advanced Topics</strong> (click to expand)</summary>
|
|
206
219
|
|
|
220
|
+
### v1.1.0 Optimizations
|
|
221
|
+
|
|
222
|
+
Version 1.1.0 introduces two major optimizations that make the tool both **faster** and **smarter**:
|
|
223
|
+
|
|
224
|
+
#### 1. Adaptive Per-Image Threshold Optimization
|
|
225
|
+
|
|
226
|
+
**Problem (v1.0.x):** A fixed threshold (1.42) was used for all images, but optimal thresholds vary by image quality, ice thickness, and lattice order.
|
|
227
|
+
|
|
228
|
+
**Solution (v1.1.0):** Automatic per-image optimization using grid search:
|
|
229
|
+
- Tests 21 threshold values in range [1.40, 1.60] with 0.01 step
|
|
230
|
+
- Scores each using a quality function that balances:
|
|
231
|
+
- **Peak count** (target: ~600 peaks for good lattice coverage)
|
|
232
|
+
- **Peak SNR** (signal-to-noise of detected peaks)
|
|
233
|
+
- **Peak distribution** (uniform hexagonal spacing preferred)
|
|
234
|
+
- **Coverage** (adequate sampling across frequency space)
|
|
235
|
+
- Returns the threshold with highest quality score
|
|
236
|
+
|
|
237
|
+
**Result:** Each image gets its optimal threshold automatically.
|
|
238
|
+
|
|
239
|
+

|
|
240
|
+
|
|
241
|
+
#### 2. GPU-Accelerated Background Subtraction (Kornia)
|
|
242
|
+
|
|
243
|
+
**Problem (v1.0.x):** Profiling revealed background subtraction (scipy median filter) consumed **94% of processing time** — not FFT as expected.
|
|
244
|
+
|
|
245
|
+
**Solution (v1.1.0):** Replaced scipy's CPU median filter with Kornia's GPU implementation:
|
|
246
|
+
```python
|
|
247
|
+
# Before (v1.0.x) - CPU, ~2.5s per image
|
|
248
|
+
from scipy.ndimage import median_filter
|
|
249
|
+
background = median_filter(log_power, size=51)
|
|
250
|
+
|
|
251
|
+
# After (v1.1.0) - GPU, ~0.05s per image
|
|
252
|
+
from kornia.filters import median_blur
|
|
253
|
+
background = median_blur(log_power_tensor, (51, 51))
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
**Result:** 48x speedup on background subtraction alone.
|
|
257
|
+
|
|
258
|
+
#### Performance Comparison
|
|
259
|
+
|
|
260
|
+
| Version | Threshold | Background Sub | Time/Image | Quality |
|
|
261
|
+
|---------|-----------|----------------|------------|---------|
|
|
262
|
+
| v1.0.10 | Fixed (1.42) | scipy CPU | ~12s | Good |
|
|
263
|
+
| v1.1.0 | Fixed | Kornia GPU | ~1.0s | Good |
|
|
264
|
+
| v1.1.0 | **Auto** | **Kornia GPU** | **~2.6s** | **Optimal** |
|
|
265
|
+
|
|
266
|
+
**Net result:** 5x faster with better results per image.
|
|
267
|
+
|
|
268
|
+
#### Correlation Validation
|
|
269
|
+
|
|
270
|
+
Kornia GPU vs scipy CPU background subtraction correlation: **0.9976** (nearly identical output).
|
|
271
|
+
|
|
272
|
+
---
|
|
273
|
+
|
|
207
274
|
### Algorithm Details
|
|
208
275
|
|
|
209
276
|
```
|
|
@@ -224,12 +291,13 @@ The algorithm:
|
|
|
224
291
|
| Parameter | Default | Description |
|
|
225
292
|
|-----------|---------|-------------|
|
|
226
293
|
| `pixel_ang` | *required* | Pixel size in Ångstroms |
|
|
227
|
-
| `threshold` |
|
|
294
|
+
| `threshold` | **auto** | Peak detection threshold - auto-optimized per image (range 1.40-1.60) |
|
|
228
295
|
| `inside_radius_ang` | 90 | Inner resolution limit (Å) - protects structural info |
|
|
229
296
|
| `outside_radius_ang` | auto | Outer resolution limit (Å) - protects near Nyquist |
|
|
230
297
|
| `expand_pixel` | 10 | Morphological expansion of peak mask (pixels) |
|
|
231
298
|
| `unit_cell_ang` | 116 | Crystal unit cell for inpaint shift calculation (Å) |
|
|
232
299
|
| `backend` | auto | `"auto"`, `"numpy"` (CPU), or `"pytorch"` (GPU) |
|
|
300
|
+
| `use_kornia` | **true** | Use Kornia for GPU-accelerated background subtraction (48x faster) |
|
|
233
301
|
|
|
234
302
|
### Supported Hardware
|
|
235
303
|
|
|
@@ -254,16 +322,17 @@ pytest tests/ -v # Run tests
|
|
|
254
322
|
|
|
255
323
|
```
|
|
256
324
|
src/lattice_subtraction/
|
|
257
|
-
├── __init__.py
|
|
258
|
-
├── cli.py
|
|
259
|
-
├── core.py
|
|
260
|
-
├── batch.py
|
|
261
|
-
├── config.py
|
|
262
|
-
├── io.py
|
|
263
|
-
├── masks.py
|
|
264
|
-
├── processing.py
|
|
265
|
-
├──
|
|
266
|
-
|
|
325
|
+
├── __init__.py # Package exports
|
|
326
|
+
├── cli.py # Command-line interface
|
|
327
|
+
├── core.py # LatticeSubtractor main class
|
|
328
|
+
├── batch.py # Parallel batch processing
|
|
329
|
+
├── config.py # Configuration dataclass
|
|
330
|
+
├── io.py # MRC file I/O
|
|
331
|
+
├── masks.py # FFT mask generation
|
|
332
|
+
├── processing.py # FFT helpers + GPU background subtraction
|
|
333
|
+
├── threshold_optimizer.py # Auto-threshold optimization (NEW in v1.1)
|
|
334
|
+
├── ui.py # Terminal UI
|
|
335
|
+
└── visualization.py # Comparison figures
|
|
267
336
|
```
|
|
268
337
|
|
|
269
338
|
### Migration from MATLAB
|
|
Binary file
|
|
Binary file
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
# Lattice Subtraction - Example Configuration
|
|
1
|
+
# Lattice Subtraction - Example Configuration (v1.1.0)
|
|
2
2
|
#
|
|
3
3
|
# This file contains all available configuration options.
|
|
4
4
|
# Copy and modify for your specific dataset.
|
|
5
|
+
#
|
|
6
|
+
# NEW in v1.1.0: Auto-threshold and Kornia GPU acceleration are now defaults!
|
|
7
|
+
# Just run `lattice-sub process image.mrc -p 0.56` for optimal results.
|
|
5
8
|
|
|
6
9
|
# ============================================
|
|
7
10
|
# REQUIRED: Detector/Pixel Size
|
|
@@ -16,9 +19,11 @@ pixel_ang: 0.56
|
|
|
16
19
|
# ============================================
|
|
17
20
|
|
|
18
21
|
# Threshold for peak detection on log-amplitude FFT
|
|
22
|
+
# Options:
|
|
23
|
+
# - "auto" (default): Optimizes threshold per image in range [1.40, 1.60]
|
|
24
|
+
# - A fixed value (e.g., 1.42): Uses same threshold for all images
|
|
19
25
|
# Higher values = more conservative (fewer peaks detected)
|
|
20
|
-
|
|
21
|
-
threshold: 1.42
|
|
26
|
+
threshold: auto
|
|
22
27
|
|
|
23
28
|
# ============================================
|
|
24
29
|
# Resolution Limits (in Angstroms)
|
|
@@ -69,3 +74,12 @@ pad_output: false
|
|
|
69
74
|
# Backend for FFT computation
|
|
70
75
|
# Options: "auto" (default, GPU if available), "numpy" (CPU), or "pytorch" (GPU)
|
|
71
76
|
backend: auto
|
|
77
|
+
|
|
78
|
+
# ============================================
|
|
79
|
+
# GPU Acceleration (NEW in v1.1.0)
|
|
80
|
+
# ============================================
|
|
81
|
+
|
|
82
|
+
# Use Kornia for GPU-accelerated background subtraction
|
|
83
|
+
# This provides ~48x speedup over scipy's CPU median filter
|
|
84
|
+
# Set to false to use CPU scipy (for debugging or CPU-only systems)
|
|
85
|
+
use_kornia: true
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "lattice-sub"
|
|
7
|
-
version = "1.0
|
|
7
|
+
version = "1.1.0"
|
|
8
8
|
description = "Lattice subtraction for cryo-EM micrographs - removes periodic crystal signals to reveal non-periodic features"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
@@ -34,6 +34,7 @@ dependencies = [
|
|
|
34
34
|
"scikit-image>=0.21",
|
|
35
35
|
"torch>=2.0",
|
|
36
36
|
"matplotlib>=3.7",
|
|
37
|
+
"kornia>=0.7",
|
|
37
38
|
]
|
|
38
39
|
|
|
39
40
|
[project.optional-dependencies]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lattice-sub
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Lattice subtraction for cryo-EM micrographs - removes periodic crystal signals to reveal non-periodic features
|
|
5
5
|
Author-email: George Stephenson <george.stephenson@colorado.edu>, Vignesh Kasinath <vignesh.kasinath@colorado.edu>
|
|
6
6
|
License: MIT
|
|
@@ -28,6 +28,7 @@ Requires-Dist: click>=8.1
|
|
|
28
28
|
Requires-Dist: scikit-image>=0.21
|
|
29
29
|
Requires-Dist: torch>=2.0
|
|
30
30
|
Requires-Dist: matplotlib>=3.7
|
|
31
|
+
Requires-Dist: kornia>=0.7
|
|
31
32
|
Provides-Extra: dev
|
|
32
33
|
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
33
34
|
Requires-Dist: pytest-cov; extra == "dev"
|
|
@@ -67,6 +68,8 @@ pip install lattice-sub
|
|
|
67
68
|
|
|
68
69
|
That's it! GPU acceleration works automatically if you have an NVIDIA GPU.
|
|
69
70
|
|
|
71
|
+
> **Note:** Requires Python 3.11+ and an NVIDIA GPU (RTX 20/30/40 series, A100, etc.) for best performance. CPU fallback is available but slower.
|
|
72
|
+
|
|
70
73
|
---
|
|
71
74
|
|
|
72
75
|
## Quick Start
|
|
@@ -77,6 +80,8 @@ That's it! GPU acceleration works automatically if you have an NVIDIA GPU.
|
|
|
77
80
|
lattice-sub process your_image.mrc -o output.mrc --pixel-size 0.56
|
|
78
81
|
```
|
|
79
82
|
|
|
83
|
+
> **Pixel size:** Use your detector's actual pixel size (e.g., K3=0.56Å, Falcon=1.14Å, K2=1.32Å)
|
|
84
|
+
|
|
80
85
|
### Process a Folder of Images
|
|
81
86
|
|
|
82
87
|
```bash
|
|
@@ -99,7 +104,7 @@ This creates side-by-side PNG images showing before/after/difference for each mi
|
|
|
99
104
|
|--------|-------------|
|
|
100
105
|
| `-p, --pixel-size` | **Required.** Pixel size in Ångstroms |
|
|
101
106
|
| `-o, --output` | Output file path (default: `sub_<input>`) |
|
|
102
|
-
| `-t, --threshold` | Peak detection sensitivity (default:
|
|
107
|
+
| `-t, --threshold` | Peak detection sensitivity (default: **auto** - optimized per image) |
|
|
103
108
|
| `--cpu` | Force CPU processing (GPU is used by default) |
|
|
104
109
|
| `-q, --quiet` | Hide the banner and progress messages |
|
|
105
110
|
| `-v, --verbose` | Show detailed processing information |
|
|
@@ -107,7 +112,10 @@ This creates side-by-side PNG images showing before/after/difference for each mi
|
|
|
107
112
|
### Example with Options
|
|
108
113
|
|
|
109
114
|
```bash
|
|
110
|
-
# Process with
|
|
115
|
+
# Process with auto-optimized threshold (default - recommended)
|
|
116
|
+
lattice-sub process image.mrc -o cleaned.mrc -p 0.56 -v
|
|
117
|
+
|
|
118
|
+
# Override with a specific threshold if needed
|
|
111
119
|
lattice-sub process image.mrc -o cleaned.mrc -p 0.56 -t 1.5 -v
|
|
112
120
|
|
|
113
121
|
# Batch process, force CPU with 8 parallel workers
|
|
@@ -116,16 +124,22 @@ lattice-sub batch raw/ processed/ -p 0.56 --cpu -j 8
|
|
|
116
124
|
|
|
117
125
|
---
|
|
118
126
|
|
|
119
|
-
## Using a Config File
|
|
127
|
+
## Using a Config File (Optional)
|
|
128
|
+
|
|
129
|
+
For most users, the defaults work great — just use `-p` for pixel size:
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
lattice-sub batch input/ output/ -p 0.56
|
|
133
|
+
```
|
|
120
134
|
|
|
121
|
-
|
|
135
|
+
Config files are useful for **reproducibility** or **non-standard samples**:
|
|
122
136
|
|
|
123
137
|
```yaml
|
|
124
|
-
# params.yaml
|
|
138
|
+
# params.yaml - only include what you want to override
|
|
125
139
|
pixel_ang: 0.56
|
|
126
|
-
|
|
127
|
-
inside_radius_ang: 90
|
|
128
|
-
|
|
140
|
+
unit_cell_ang: 120 # Default: 116 (nucleosome). Change for other crystals.
|
|
141
|
+
inside_radius_ang: 80 # Default: 90. Protect different resolution range.
|
|
142
|
+
# threshold: 1.45 # Uncomment to override auto-optimization
|
|
129
143
|
```
|
|
130
144
|
|
|
131
145
|
Then use it:
|
|
@@ -244,6 +258,60 @@ MIT License - see [LICENSE](LICENSE) for details.
|
|
|
244
258
|
<details>
|
|
245
259
|
<summary><strong>📚 Advanced Topics</strong> (click to expand)</summary>
|
|
246
260
|
|
|
261
|
+
### v1.1.0 Optimizations
|
|
262
|
+
|
|
263
|
+
Version 1.1.0 introduces two major optimizations that make the tool both **faster** and **smarter**:
|
|
264
|
+
|
|
265
|
+
#### 1. Adaptive Per-Image Threshold Optimization
|
|
266
|
+
|
|
267
|
+
**Problem (v1.0.x):** A fixed threshold (1.42) was used for all images, but optimal thresholds vary by image quality, ice thickness, and lattice order.
|
|
268
|
+
|
|
269
|
+
**Solution (v1.1.0):** Automatic per-image optimization using grid search:
|
|
270
|
+
- Tests 21 threshold values in range [1.40, 1.60] with 0.01 step
|
|
271
|
+
- Scores each using a quality function that balances:
|
|
272
|
+
- **Peak count** (target: ~600 peaks for good lattice coverage)
|
|
273
|
+
- **Peak SNR** (signal-to-noise of detected peaks)
|
|
274
|
+
- **Peak distribution** (uniform hexagonal spacing preferred)
|
|
275
|
+
- **Coverage** (adequate sampling across frequency space)
|
|
276
|
+
- Returns the threshold with highest quality score
|
|
277
|
+
|
|
278
|
+
**Result:** Each image gets its optimal threshold automatically.
|
|
279
|
+
|
|
280
|
+

|
|
281
|
+
|
|
282
|
+
#### 2. GPU-Accelerated Background Subtraction (Kornia)
|
|
283
|
+
|
|
284
|
+
**Problem (v1.0.x):** Profiling revealed background subtraction (scipy median filter) consumed **94% of processing time** — not FFT as expected.
|
|
285
|
+
|
|
286
|
+
**Solution (v1.1.0):** Replaced scipy's CPU median filter with Kornia's GPU implementation:
|
|
287
|
+
```python
|
|
288
|
+
# Before (v1.0.x) - CPU, ~2.5s per image
|
|
289
|
+
from scipy.ndimage import median_filter
|
|
290
|
+
background = median_filter(log_power, size=51)
|
|
291
|
+
|
|
292
|
+
# After (v1.1.0) - GPU, ~0.05s per image
|
|
293
|
+
from kornia.filters import median_blur
|
|
294
|
+
background = median_blur(log_power_tensor, (51, 51))
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
**Result:** 48x speedup on background subtraction alone.
|
|
298
|
+
|
|
299
|
+
#### Performance Comparison
|
|
300
|
+
|
|
301
|
+
| Version | Threshold | Background Sub | Time/Image | Quality |
|
|
302
|
+
|---------|-----------|----------------|------------|---------|
|
|
303
|
+
| v1.0.10 | Fixed (1.42) | scipy CPU | ~12s | Good |
|
|
304
|
+
| v1.1.0 | Fixed | Kornia GPU | ~1.0s | Good |
|
|
305
|
+
| v1.1.0 | **Auto** | **Kornia GPU** | **~2.6s** | **Optimal** |
|
|
306
|
+
|
|
307
|
+
**Net result:** 5x faster with better results per image.
|
|
308
|
+
|
|
309
|
+
#### Correlation Validation
|
|
310
|
+
|
|
311
|
+
Kornia GPU vs scipy CPU background subtraction correlation: **0.9976** (nearly identical output).
|
|
312
|
+
|
|
313
|
+
---
|
|
314
|
+
|
|
247
315
|
### Algorithm Details
|
|
248
316
|
|
|
249
317
|
```
|
|
@@ -264,12 +332,13 @@ The algorithm:
|
|
|
264
332
|
| Parameter | Default | Description |
|
|
265
333
|
|-----------|---------|-------------|
|
|
266
334
|
| `pixel_ang` | *required* | Pixel size in Ångstroms |
|
|
267
|
-
| `threshold` |
|
|
335
|
+
| `threshold` | **auto** | Peak detection threshold - auto-optimized per image (range 1.40-1.60) |
|
|
268
336
|
| `inside_radius_ang` | 90 | Inner resolution limit (Å) - protects structural info |
|
|
269
337
|
| `outside_radius_ang` | auto | Outer resolution limit (Å) - protects near Nyquist |
|
|
270
338
|
| `expand_pixel` | 10 | Morphological expansion of peak mask (pixels) |
|
|
271
339
|
| `unit_cell_ang` | 116 | Crystal unit cell for inpaint shift calculation (Å) |
|
|
272
340
|
| `backend` | auto | `"auto"`, `"numpy"` (CPU), or `"pytorch"` (GPU) |
|
|
341
|
+
| `use_kornia` | **true** | Use Kornia for GPU-accelerated background subtraction (48x faster) |
|
|
273
342
|
|
|
274
343
|
### Supported Hardware
|
|
275
344
|
|
|
@@ -294,16 +363,17 @@ pytest tests/ -v # Run tests
|
|
|
294
363
|
|
|
295
364
|
```
|
|
296
365
|
src/lattice_subtraction/
|
|
297
|
-
├── __init__.py
|
|
298
|
-
├── cli.py
|
|
299
|
-
├── core.py
|
|
300
|
-
├── batch.py
|
|
301
|
-
├── config.py
|
|
302
|
-
├── io.py
|
|
303
|
-
├── masks.py
|
|
304
|
-
├── processing.py
|
|
305
|
-
├──
|
|
306
|
-
|
|
366
|
+
├── __init__.py # Package exports
|
|
367
|
+
├── cli.py # Command-line interface
|
|
368
|
+
├── core.py # LatticeSubtractor main class
|
|
369
|
+
├── batch.py # Parallel batch processing
|
|
370
|
+
├── config.py # Configuration dataclass
|
|
371
|
+
├── io.py # MRC file I/O
|
|
372
|
+
├── masks.py # FFT mask generation
|
|
373
|
+
├── processing.py # FFT helpers + GPU background subtraction
|
|
374
|
+
├── threshold_optimizer.py # Auto-threshold optimization (NEW in v1.1)
|
|
375
|
+
├── ui.py # Terminal UI
|
|
376
|
+
└── visualization.py # Comparison figures
|
|
307
377
|
```
|
|
308
378
|
|
|
309
379
|
### Migration from MATLAB
|
|
@@ -2,8 +2,8 @@ LICENSE
|
|
|
2
2
|
MANIFEST.in
|
|
3
3
|
README.md
|
|
4
4
|
pyproject.toml
|
|
5
|
-
docs/architecture.md
|
|
6
5
|
docs/images/example_comparison.png
|
|
6
|
+
docs/images/threshold_analysis.png
|
|
7
7
|
examples/config.yaml
|
|
8
8
|
examples/converted_params.yaml
|
|
9
9
|
src/lattice_sub.egg-info/PKG-INFO
|
|
@@ -20,6 +20,6 @@ src/lattice_subtraction/core.py
|
|
|
20
20
|
src/lattice_subtraction/io.py
|
|
21
21
|
src/lattice_subtraction/masks.py
|
|
22
22
|
src/lattice_subtraction/processing.py
|
|
23
|
+
src/lattice_subtraction/threshold_optimizer.py
|
|
23
24
|
src/lattice_subtraction/ui.py
|
|
24
|
-
src/lattice_subtraction/visualization.py
|
|
25
|
-
tests/test_lattice_subtraction.py
|
|
25
|
+
src/lattice_subtraction/visualization.py
|