emdbva 0.0.1.dev139__tar.gz → 0.0.1.dev143__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {emdbva-0.0.1.dev139/emdbva.egg-info → emdbva-0.0.1.dev143}/PKG-INFO +1 -1
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143/emdbva.egg-info}/PKG-INFO +1 -1
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/emdbva.egg-info/SOURCES.txt +2 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/setup.py +1 -1
- emdbva-0.0.1.dev143/va/PATHS_codon_airflow.py +80 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/mainva.py +14 -6
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/bars.py +40 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/connected_percentage.py +30 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/contour_level_predicator.py +38 -1
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/map_data_validation.py +123 -6
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/phaserandomization.py +30 -2
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/phenix_cc.py +18 -11
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/phenix_mm.py +14 -5
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/projections.py +72 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/qscore.py +13 -5
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/resmap.py +33 -8
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/smoc.py +11 -4
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/strudel.py +1 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/preparation.py +351 -33
- emdbva-0.0.1.dev143/va/prepareandrun_codon_airflow.py +1084 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/Checker.py +56 -1
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/ChimeraxViews.py +8 -1
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/MapProcessor.py +16 -4
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/Model.py +5 -1
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/misc.py +16 -13
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/rescolor.py +17 -1
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/stars.py +4 -1
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/validationanalysis.py +173 -43
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/version.py +1 -1
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/LICENSE +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/MANIFEST.in +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/README.rst +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/emdbva.egg-info/dependency_links.txt +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/emdbva.egg-info/entry_points.txt +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/emdbva.egg-info/requires.txt +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/emdbva.egg-info/top_level.txt +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/setup.cfg +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/__init__.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/__init__.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/emda_mmcc.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/emringer.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/inclusion.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/overlap_percentage.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/residue_locres.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/surfaces.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/metrics/threedfsc.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/qscores.csv +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/__init__.py +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/cl_weights.pth +0 -0
- {emdbva-0.0.1.dev139 → emdbva-0.0.1.dev143}/va/utils/log_utils.py +0 -0
|
@@ -8,9 +8,11 @@ emdbva.egg-info/dependency_links.txt
|
|
|
8
8
|
emdbva.egg-info/entry_points.txt
|
|
9
9
|
emdbva.egg-info/requires.txt
|
|
10
10
|
emdbva.egg-info/top_level.txt
|
|
11
|
+
va/PATHS_codon_airflow.py
|
|
11
12
|
va/__init__.py
|
|
12
13
|
va/mainva.py
|
|
13
14
|
va/preparation.py
|
|
15
|
+
va/prepareandrun_codon_airflow.py
|
|
14
16
|
va/qscores.csv
|
|
15
17
|
va/validationanalysis.py
|
|
16
18
|
va/version.py
|
|
@@ -20,7 +20,7 @@ setup(
|
|
|
20
20
|
version=__version__,
|
|
21
21
|
packages=find_packages(),
|
|
22
22
|
exclude_package_data={
|
|
23
|
-
'va': ['test/*', '
|
|
23
|
+
'va': ['test/*', 'core/*', 'proc/*', 'validation/*', 'test_data/*'],
|
|
24
24
|
},
|
|
25
25
|
include_package_data=True,
|
|
26
26
|
author="Zhe Wang",
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""
|
|
2
|
+
PATHS.py
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
PATHS is used to input and output directories of validation
|
|
6
|
+
analysis.
|
|
7
|
+
|
|
8
|
+
Copyright [2013] EMBL - European Bioinformatics Institute
|
|
9
|
+
Licensed under the Apache License, Version 2.0 (the
|
|
10
|
+
"License"); you may not use this file except in
|
|
11
|
+
compliance with the License. You may obtain a copy of
|
|
12
|
+
the License at
|
|
13
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
14
|
+
Unless required by applicable law or agreed to in writing,
|
|
15
|
+
software distributed under the License is distributed on
|
|
16
|
+
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
17
|
+
KIND, either express or implied. See the License for the
|
|
18
|
+
specific language governing permissions and limitations
|
|
19
|
+
under the License.
|
|
20
|
+
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
__author__ = 'Zhe Wang'
|
|
24
|
+
__email__ = 'zhe@ebi.ac.uk'
|
|
25
|
+
__date__ = '2022-06-14'
|
|
26
|
+
|
|
27
|
+
# ENT file source path
|
|
28
|
+
ENTSOURCE = '/nfs/msd/work2/ftp/pdb/data/structures/all/pdb/'
|
|
29
|
+
|
|
30
|
+
# VTP file source path
|
|
31
|
+
VTPSOURCE = '/nfs/pdbe_staging/pdbe_data/development/staging/em/'
|
|
32
|
+
|
|
33
|
+
# CIF file source path
|
|
34
|
+
# CIFSOURCE = '/nfs/production/gerard/pdbe/prerelease/current/ftp/pdb/data/structures/divided/mmCIF/'
|
|
35
|
+
# CIFSOURCE = '/nfs/ftp/public/.staging/pdbe-G79Bblrm/projects/pdb/staging/data/structures/all/mmCIF/'
|
|
36
|
+
# CIFSOURCE = '/hps/nobackup/pdbe/nfs_prod/ftp/data/structures/all/mmCIF/'
|
|
37
|
+
CIFSOURCE = '/nfs/production/gerard/pdbe/ftp/pdb/data/structures/all/mmCIF/'
|
|
38
|
+
|
|
39
|
+
MAP_SERVER_PATH = '/hps/nobackup/gerard/emdb/va/entry_results/'
|
|
40
|
+
|
|
41
|
+
# Chimera path
|
|
42
|
+
# CHIMERA = '/hps/nobackup/gerard/emdb/va/external/chimerax/usr/libexec/UCSF-ChimeraX/bin/ChimeraX'
|
|
43
|
+
CHIMERA = '/hps/nobackup/gerard/emdb/va/external/chimera110/usr/libexec/UCSF-ChimeraX/bin/ChimeraX'
|
|
44
|
+
|
|
45
|
+
# Original or old Chimera
|
|
46
|
+
OCHIMERA = '/hps/nobackup/gerard/emdb/va/external/chimera/bin/chimera'
|
|
47
|
+
|
|
48
|
+
# For ebi server to copy data for VA
|
|
49
|
+
# VASOURCE = '/nfs/msd/em/ftp_rsync/staging/structures/'
|
|
50
|
+
# VASOURCE = '/nfs/production/gerard/emdb/archive/staging/structures/'
|
|
51
|
+
# VASOURCE = '/hps/nobackup/gerard/emdb/archive/local/structures/'
|
|
52
|
+
VASOURCE = '/nfs/production/gerard/emdb/archive2/local/structures/'
|
|
53
|
+
|
|
54
|
+
# VAPATH
|
|
55
|
+
VAPATH = '/hps/nobackup/gerard/emdb/va_airflow/Validation-Analysis/va/'
|
|
56
|
+
|
|
57
|
+
# Jsons and images are copied to here for display purpose
|
|
58
|
+
FORDISPLAY = '/nfs/nobackup/msd/em_va/development/'
|
|
59
|
+
|
|
60
|
+
# Proshade path
|
|
61
|
+
PROSHADEPATH = '/hps/software/users/gerard/emdb/va/external/ccpem/ccpem-1.6.0/bin/proshade'
|
|
62
|
+
|
|
63
|
+
# Meshmaker path
|
|
64
|
+
MESHMAKERPATH = '/nfs/msd/em/software/meshmakertest/meshmaker/build/meshmaker'
|
|
65
|
+
|
|
66
|
+
# VA production path
|
|
67
|
+
VA_PROD_PATH = '/nfs/public/rw/pdbe/httpd-em/sessions-prod/validation_analysis'
|
|
68
|
+
|
|
69
|
+
# VA staging path
|
|
70
|
+
VA_STAG_PATH = '/nfs/public/rw/pdbe/httpd-em/sessions/validation_analysis'
|
|
71
|
+
|
|
72
|
+
# Strudel lib
|
|
73
|
+
LIB_STRUDEL_ROOT = '/hps/nobackup/gerard/emdb/va/external/strudel_motiflib/strudel-lib.4_0.5px'
|
|
74
|
+
|
|
75
|
+
# 3DFSC fake to take positions
|
|
76
|
+
THREEDFSC_ROOT = '/Users/zhe/Downloads/Anisotropy/ThreeDFSC/ThreeDFSC_Start.py'
|
|
77
|
+
|
|
78
|
+
# ResMap
|
|
79
|
+
# RESMAP = '/hps/nobackup/gerard/emdb/va/external/resmap/ResMap_project/ResMap.py'
|
|
80
|
+
RESMAP = '/hps/software/users/gerard/emdb/va/external/ccpem/ccpem-1.6.0/bin/relion_postprocess'
|
|
@@ -3,8 +3,10 @@
|
|
|
3
3
|
"""
|
|
4
4
|
mainva.py
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
Entry points and orchestration helpers for the main VA pipeline.
|
|
7
|
+
|
|
8
|
+
This module wires together preparation, validation analysis, and
|
|
9
|
+
logging for the standard package-level VA workflow.
|
|
8
10
|
|
|
9
11
|
Copyright [2013] EMBL - European Bioinformatics Institute
|
|
10
12
|
Licensed under the Apache License, Version 2.0 (the
|
|
@@ -121,9 +123,9 @@ def allruns(validationobj, runs):
|
|
|
121
123
|
if 'symmetry' in runs:
|
|
122
124
|
validationobj.symmetry()
|
|
123
125
|
|
|
124
|
-
# Strudel
|
|
125
|
-
if 'strudel' in runs:
|
|
126
|
-
|
|
126
|
+
# Strudel (turning off)
|
|
127
|
+
# if 'strudel' in runs:
|
|
128
|
+
# validationobj.strudel()
|
|
127
129
|
|
|
128
130
|
# Q-score
|
|
129
131
|
if 'qscore' in runs:
|
|
@@ -170,6 +172,7 @@ def inallruns(validationobj, runs):
|
|
|
170
172
|
"""
|
|
171
173
|
|
|
172
174
|
def without_strudel():
|
|
175
|
+
"""Run the selected validation steps other than Strudel."""
|
|
173
176
|
# Projections
|
|
174
177
|
if 'projection' in runs:
|
|
175
178
|
validationobj.orthogonal_projections()
|
|
@@ -255,6 +258,7 @@ def inallruns(validationobj, runs):
|
|
|
255
258
|
return None
|
|
256
259
|
|
|
257
260
|
def strudel_only():
|
|
261
|
+
"""Run the selected Strudel validation step."""
|
|
258
262
|
# Strudel
|
|
259
263
|
if 'strudel' in runs:
|
|
260
264
|
validationobj.strudel()
|
|
@@ -333,6 +337,11 @@ def reconfigure_logging(temp_logger, buffer, log_file):
|
|
|
333
337
|
|
|
334
338
|
# @profile
|
|
335
339
|
def main():
|
|
340
|
+
"""Run the main VA command-line entry point.
|
|
341
|
+
|
|
342
|
+
This function parses the CLI arguments, builds the preparation and
|
|
343
|
+
validation objects, and drives the configured validation runs.
|
|
344
|
+
"""
|
|
336
345
|
|
|
337
346
|
start_first = timeit.default_timer()
|
|
338
347
|
|
|
@@ -443,4 +452,3 @@ def main():
|
|
|
443
452
|
|
|
444
453
|
if __name__ == '__main__':
|
|
445
454
|
main()
|
|
446
|
-
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
"""Calculate score percentiles and generate metric comparison plots."""
|
|
2
|
+
|
|
1
3
|
import os.path
|
|
2
4
|
import json
|
|
3
5
|
import pandas as pd
|
|
@@ -317,6 +319,22 @@ def plot_bar_mat(a, b, qmin, qmax, qscore, work_dir, plot_name, score_type, resb
|
|
|
317
319
|
|
|
318
320
|
|
|
319
321
|
def bar(new_entry_dict, score_type, work_dir, score_dir, plot_name, update_bin_file=None):
|
|
322
|
+
"""Calculate whole-dataset and resolution-relative score percentiles.
|
|
323
|
+
|
|
324
|
+
Args:
|
|
325
|
+
new_entry_dict: Score entry containing the selected score and resolution.
|
|
326
|
+
score_type: Score column to process.
|
|
327
|
+
work_dir: Directory for generated plot output.
|
|
328
|
+
score_dir: Directory containing the default score CSV.
|
|
329
|
+
plot_name: Plot filename used when a bar plot is generated.
|
|
330
|
+
update_bin_file: Optional CSV used to update the resolution-bin size.
|
|
331
|
+
|
|
332
|
+
Returns:
|
|
333
|
+
tuple: Whole-dataset statistics, relative-resolution statistics, and bin size.
|
|
334
|
+
|
|
335
|
+
Raises:
|
|
336
|
+
ValueError: If no usable score CSV is available.
|
|
337
|
+
"""
|
|
320
338
|
if update_bin_file and os.path.isfile(update_bin_file):
|
|
321
339
|
input_file = update_bin_file
|
|
322
340
|
else:
|
|
@@ -391,6 +409,11 @@ def score_whole(new_entry, df, score_type='qscore'):
|
|
|
391
409
|
|
|
392
410
|
|
|
393
411
|
def score_relative(new_entry, df, score_type='qscore', column='resolution', bin=0.5):
|
|
412
|
+
"""Calculate a score percentile within a resolution range.
|
|
413
|
+
|
|
414
|
+
Returns:
|
|
415
|
+
tuple: Relative percentile and number of rows in the selected range.
|
|
416
|
+
"""
|
|
394
417
|
df_relative = get_resolution_range(new_entry, df, score_type, column, bin)
|
|
395
418
|
if df_relative.empty: # Handle case where df_relative is empty
|
|
396
419
|
return 0., 0. # Return NaN for q_relative and an empty list for row_indices
|
|
@@ -401,10 +424,16 @@ def score_relative(new_entry, df, score_type='qscore', column='resolution', bin=
|
|
|
401
424
|
return q_relative, rows
|
|
402
425
|
|
|
403
426
|
def score_relative_wrapper(args):
|
|
427
|
+
"""Unpack arguments for parallel relative-score calculation."""
|
|
404
428
|
x, df, score_type, column, bin = args
|
|
405
429
|
return score_relative(x, df, score_type, column, bin)
|
|
406
430
|
|
|
407
431
|
def fast_qscore_resolution_bin(df_resolution_sorted, score_type='qscore'):
|
|
432
|
+
"""Add relative-score columns for a series of resolution-bin sizes.
|
|
433
|
+
|
|
434
|
+
Returns:
|
|
435
|
+
pandas.DataFrame: The input DataFrame with calculated relative-score columns.
|
|
436
|
+
"""
|
|
408
437
|
values = [round(x, 1) for x in np.arange(0.1, 1.6, 0.1)]
|
|
409
438
|
for value in values:
|
|
410
439
|
args_list = [
|
|
@@ -417,6 +446,11 @@ def fast_qscore_resolution_bin(df_resolution_sorted, score_type='qscore'):
|
|
|
417
446
|
return df_resolution_sorted
|
|
418
447
|
|
|
419
448
|
def find_optimal_correlation_index(arr):
|
|
449
|
+
"""Return the index of the value closest to -0.3 among eligible values.
|
|
450
|
+
|
|
451
|
+
Returns:
|
|
452
|
+
int or None: Selected index, or ``None`` if no value exceeds -0.3.
|
|
453
|
+
"""
|
|
420
454
|
arr = np.asarray(arr)
|
|
421
455
|
mask = arr > -0.3
|
|
422
456
|
if not np.any(mask):
|
|
@@ -426,6 +460,11 @@ def find_optimal_correlation_index(arr):
|
|
|
426
460
|
return np.where(arr == closest_value)[0][0]
|
|
427
461
|
|
|
428
462
|
def resolution_qrelative_correlation(df_resolution_sorted, values, score_type='qscore'):
|
|
463
|
+
"""Calculate correlations between resolution and relative-score columns.
|
|
464
|
+
|
|
465
|
+
Returns:
|
|
466
|
+
tuple: Correlations below 5, correlations above 5, and column names.
|
|
467
|
+
"""
|
|
429
468
|
correlation_below_5 = []
|
|
430
469
|
correlation_above_5 = []
|
|
431
470
|
col_names = []
|
|
@@ -463,6 +502,7 @@ def resolution_qrelative_correlation(df_resolution_sorted, values, score_type='q
|
|
|
463
502
|
return correlation_below_5, correlation_above_5, col_names
|
|
464
503
|
|
|
465
504
|
def get_resolution_bin_size_fromva(score_file):
|
|
505
|
+
"""Read a score CSV and return its recorded resolution-bin mode."""
|
|
466
506
|
df = pd.read_csv(score_file, dtype={'id': str, 'resolution': float, 'name': str, 'qscore': float}, usecols=[0, 1, 2, 3])
|
|
467
507
|
df = df.dropna()
|
|
468
508
|
df_resolution_sorted = df.sort_values(by='resolution')
|
|
@@ -1,12 +1,34 @@
|
|
|
1
|
+
"""Measure connected and removed regions in thresholded MRC volumes."""
|
|
2
|
+
|
|
1
3
|
import numpy as np
|
|
2
4
|
import mrcfile
|
|
3
5
|
from scipy.ndimage import label, find_objects
|
|
4
6
|
|
|
5
7
|
|
|
6
8
|
def estimate_diameter_bbox(coords_physical):
|
|
9
|
+
"""Estimate a point set's diameter from its axis-aligned bounding box.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
coords_physical: Array of physical-space coordinates.
|
|
13
|
+
|
|
14
|
+
Returns:
|
|
15
|
+
float: Euclidean distance between opposite bounding-box corners.
|
|
16
|
+
"""
|
|
7
17
|
return np.linalg.norm(coords_physical.max(axis=0) - coords_physical.min(axis=0))
|
|
8
18
|
|
|
9
19
|
def detect_small_blobs(volume, threshold, min_diameter, voxel_size=1.0, connectivity=2):
|
|
20
|
+
"""Keep thresholded connected regions whose bounding-box diameter is large enough.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
volume: Three-dimensional volume array.
|
|
24
|
+
threshold: Value used to create the binary foreground mask.
|
|
25
|
+
min_diameter: Minimum physical diameter to retain.
|
|
26
|
+
voxel_size: Physical size of one voxel.
|
|
27
|
+
connectivity: Connectivity mode used for labeling.
|
|
28
|
+
|
|
29
|
+
Returns:
|
|
30
|
+
tuple: Cleaned boolean mask and the number of removed voxels.
|
|
31
|
+
"""
|
|
10
32
|
binary = volume > threshold
|
|
11
33
|
structure = np.ones((3, 3, 3)) if connectivity == 3 else None
|
|
12
34
|
labeled, _ = label(binary, structure=structure)
|
|
@@ -26,6 +48,14 @@ def detect_small_blobs(volume, threshold, min_diameter, voxel_size=1.0, connecti
|
|
|
26
48
|
return cleaned, total_removed_voxels
|
|
27
49
|
|
|
28
50
|
def compute_connected_metrics(mrc_path, threshold, min_diameter=5.0, voxel_size=None, connectivity=2):
|
|
51
|
+
"""Calculate connected and disconnected volume percentages from an MRC file.
|
|
52
|
+
|
|
53
|
+
The MRC volume is read from ``mrc_path``; the returned metrics are calculated
|
|
54
|
+
after removing regions below the requested diameter.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
dict: Connected and disconnected percentages and volumes.
|
|
58
|
+
"""
|
|
29
59
|
with mrcfile.open(mrc_path, permissive=True) as mrc:
|
|
30
60
|
volume = mrc.data.copy()
|
|
31
61
|
if voxel_size is None:
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
"""Predict contour levels from normalized three-dimensional map volumes."""
|
|
2
|
+
|
|
1
3
|
import os
|
|
2
4
|
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
|
|
3
5
|
import torch
|
|
@@ -14,7 +16,10 @@ warnings.filterwarnings("ignore", message="User provided device_type of 'cuda',
|
|
|
14
16
|
|
|
15
17
|
# Convolutional layers adapted from https://www.nature.com/articles/s41598-022-19212-6
|
|
16
18
|
class CNNModel(nn.Module):
|
|
19
|
+
"""Three-dimensional convolutional model used for contour prediction."""
|
|
20
|
+
|
|
17
21
|
def __init__(self):
|
|
22
|
+
"""Initialize the convolutional and fully connected layers."""
|
|
18
23
|
super(CNNModel, self).__init__()
|
|
19
24
|
self.conv = nn.Sequential(
|
|
20
25
|
nn.Conv3d(1, 32, 7, 1, padding_mode='replicate'),
|
|
@@ -42,6 +47,7 @@ class CNNModel(nn.Module):
|
|
|
42
47
|
nn.Linear(in_features=64, out_features=1))
|
|
43
48
|
|
|
44
49
|
def forward(self, x):
|
|
50
|
+
"""Run a forward pass on a batch of 64-by-64-by-64 volumes."""
|
|
45
51
|
x = x.view(-1, 1, 64, 64, 64)
|
|
46
52
|
x = self.conv(x)
|
|
47
53
|
x = x.view(x.size(0), -1)
|
|
@@ -50,10 +56,19 @@ class CNNModel(nn.Module):
|
|
|
50
56
|
|
|
51
57
|
|
|
52
58
|
def check_boxsize(vol):
|
|
59
|
+
"""Return the first dimension of a volume array."""
|
|
53
60
|
return vol.shape[0]
|
|
54
61
|
|
|
55
62
|
|
|
56
63
|
def load_data(vol_d_norm):
|
|
64
|
+
"""Wrap a normalized volume in a single-item PyTorch data loader.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
vol_d_norm: Normalized three-dimensional volume.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
DataLoader: Single-item loader for model prediction.
|
|
71
|
+
"""
|
|
57
72
|
vol_array = np.empty(shape=(1, 64, 64, 64), dtype=np.float32)
|
|
58
73
|
vol_array[:] = vol_d_norm
|
|
59
74
|
data_generator = torch.utils.data.DataLoader(vol_array, batch_size=1, shuffle=False)
|
|
@@ -61,6 +76,7 @@ def load_data(vol_d_norm):
|
|
|
61
76
|
|
|
62
77
|
|
|
63
78
|
def downsample(vol):
|
|
79
|
+
"""Downsample a volume to 64 cubed voxels in Fourier space."""
|
|
64
80
|
boxsize = check_boxsize(vol)
|
|
65
81
|
vol_ft = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(vol)))
|
|
66
82
|
start = boxsize // 2 - 64 // 2
|
|
@@ -75,6 +91,7 @@ def downsample(vol):
|
|
|
75
91
|
|
|
76
92
|
def upsample(
|
|
77
93
|
vol): # through interpolation of maps in Fourier space; based on https://www.sciencedirect.com/science/article/pii/S001046551400335X
|
|
94
|
+
"""Pad a volume in Fourier space to the model's 64 cubed input size."""
|
|
78
95
|
boxsize = check_boxsize(vol)
|
|
79
96
|
new_box_size = 64
|
|
80
97
|
vol_ft = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(vol)))
|
|
@@ -87,6 +104,11 @@ def upsample(
|
|
|
87
104
|
|
|
88
105
|
|
|
89
106
|
def normalize(vol_data):
|
|
107
|
+
"""Clip and normalize positive volume values for model input.
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
tuple: Normalized volume and the lower and upper scaling values.
|
|
111
|
+
"""
|
|
90
112
|
d_upper = np.percentile(vol_data, 99.999)
|
|
91
113
|
d_lower = np.percentile(vol_data, 0.001)
|
|
92
114
|
vol_data = np.where(vol_data > d_upper, d_upper, vol_data)
|
|
@@ -109,6 +131,14 @@ model.load_state_dict(checkpoint['model_state_dict'])
|
|
|
109
131
|
|
|
110
132
|
|
|
111
133
|
def model_pred(data_generator):
|
|
134
|
+
"""Run the loaded model over a data loader without gradient tracking.
|
|
135
|
+
|
|
136
|
+
Args:
|
|
137
|
+
data_generator: Data loader containing normalized volume batches.
|
|
138
|
+
|
|
139
|
+
Returns:
|
|
140
|
+
list: Model prediction tensors collected for each batch.
|
|
141
|
+
"""
|
|
112
142
|
model.eval()
|
|
113
143
|
pred_list = []
|
|
114
144
|
with torch.no_grad():
|
|
@@ -122,6 +152,14 @@ def model_pred(data_generator):
|
|
|
122
152
|
|
|
123
153
|
|
|
124
154
|
def calc_level_dev(vol_data):
|
|
155
|
+
"""Predict contour levels after resizing the volume to model dimensions.
|
|
156
|
+
|
|
157
|
+
Args:
|
|
158
|
+
vol_data: Three-dimensional volume array.
|
|
159
|
+
|
|
160
|
+
Returns:
|
|
161
|
+
numpy.ndarray: Predicted contour-level values rescaled to input units.
|
|
162
|
+
"""
|
|
125
163
|
boxsize = check_boxsize(vol_data)
|
|
126
164
|
|
|
127
165
|
if boxsize > 64:
|
|
@@ -154,4 +192,3 @@ def calc_level_dev(vol_data):
|
|
|
154
192
|
|
|
155
193
|
|
|
156
194
|
|
|
157
|
-
|
|
@@ -59,13 +59,127 @@ def _to_float(value):
|
|
|
59
59
|
|
|
60
60
|
|
|
61
61
|
def _parse_validation_line(line, source="mrcfile.validate"):
|
|
62
|
-
"""
|
|
63
|
-
Parse one line from mrcfile.validate() or one RuntimeWarning.
|
|
64
62
|
|
|
65
|
-
Any message not recognised by the parser is still kept in the JSON as
|
|
66
|
-
"unclassified_mrcfile_validation_message" so future mrcfile versions do
|
|
67
|
-
not silently lose information.
|
|
68
63
|
"""
|
|
64
|
+
Parse one line from ``mrcfile.validate()`` output or one captured
|
|
65
|
+
``RuntimeWarning`` into a structured validation issue.
|
|
66
|
+
The returned issue is created by ``_make_issue()`` and contains:
|
|
67
|
+
code
|
|
68
|
+
Stable machine-readable issue code.
|
|
69
|
+
category
|
|
70
|
+
Header/data/file area affected by the issue.
|
|
71
|
+
severity
|
|
72
|
+
Local VA severity assigned to the issue. This is intentionally
|
|
73
|
+
not always identical to the strict ``mrcfile.validate()`` result.
|
|
74
|
+
VA uses the parsed severity to decide its own pipeline-level
|
|
75
|
+
validity.
|
|
76
|
+
source
|
|
77
|
+
Message source, usually ``"mrcfile.validate"`` or
|
|
78
|
+
``"RuntimeWarning"``.
|
|
79
|
+
message
|
|
80
|
+
Original mrcfile message, preserved verbatim for debugging.
|
|
81
|
+
details
|
|
82
|
+
Parsed values extracted from the message, such as header field
|
|
83
|
+
names, expected values, actual values, byte counts, or exception
|
|
84
|
+
information.
|
|
85
|
+
Severity mapping used by this parser:
|
|
86
|
+
map_id_incorrect
|
|
87
|
+
warning
|
|
88
|
+
The MAP ID header string differs from the expected value.
|
|
89
|
+
map_id_missing_or_corrupt
|
|
90
|
+
warning
|
|
91
|
+
The MAP ID string is absent, suggesting the file may not be an
|
|
92
|
+
MRC file or may be corrupt.
|
|
93
|
+
machine_stamp_invalid
|
|
94
|
+
error
|
|
95
|
+
The machine stamp is invalid.
|
|
96
|
+
machine_stamp_byte_order_mismatch
|
|
97
|
+
warning
|
|
98
|
+
The machine stamp does not match the apparent byte order.
|
|
99
|
+
mode_invalid
|
|
100
|
+
error
|
|
101
|
+
The MRC mode value is invalid.
|
|
102
|
+
mode_unrecognised_data_unreadable
|
|
103
|
+
error
|
|
104
|
+
The MRC mode is unrecognised and the data block cannot be read.
|
|
105
|
+
header_field_negative
|
|
106
|
+
error
|
|
107
|
+
A non-negative integer header field such as nx, ny, nz, mx, my,
|
|
108
|
+
mz, ispg, or nlabl is negative.
|
|
109
|
+
cell_dimension_negative
|
|
110
|
+
error
|
|
111
|
+
One of the cell dimensions x, y, or z is negative.
|
|
112
|
+
axis_mapping_invalid
|
|
113
|
+
error
|
|
114
|
+
The map axis mapping is not the expected [1, 2, 3] permutation.
|
|
115
|
+
volume_stack_dimensions_invalid
|
|
116
|
+
error
|
|
117
|
+
For a volume stack, nz is not divisible by mz.
|
|
118
|
+
header_labels_empty_between_text
|
|
119
|
+
error
|
|
120
|
+
Empty labels appear between labels containing text.
|
|
121
|
+
header_labels_nlabl_mismatch
|
|
122
|
+
error
|
|
123
|
+
The nlabl header value does not match the number of non-empty
|
|
124
|
+
label records.
|
|
125
|
+
mrc_format_version_invalid
|
|
126
|
+
warning
|
|
127
|
+
The nversion field does not declare MRC2014 version 20140 or
|
|
128
|
+
20141.
|
|
129
|
+
extended_header_type_invalid
|
|
130
|
+
error
|
|
131
|
+
The extended-header type is undefined or unrecognised.
|
|
132
|
+
data_statistics_rms_mismatch
|
|
133
|
+
error
|
|
134
|
+
The calculated RMS deviation differs from the header value.
|
|
135
|
+
data_statistics_minimum_mismatch
|
|
136
|
+
error
|
|
137
|
+
The calculated minimum differs from the header value.
|
|
138
|
+
data_statistics_maximum_mismatch
|
|
139
|
+
error
|
|
140
|
+
The calculated maximum differs from the header value.
|
|
141
|
+
data_statistics_mean_mismatch
|
|
142
|
+
error
|
|
143
|
+
The calculated mean differs from the header value.
|
|
144
|
+
file_size_larger_than_expected
|
|
145
|
+
error
|
|
146
|
+
The physical file size is larger than the size calculated from
|
|
147
|
+
the header.
|
|
148
|
+
data_block_unreadable_file_size_not_checked
|
|
149
|
+
error
|
|
150
|
+
The data block could not be read, so file size could not be
|
|
151
|
+
checked.
|
|
152
|
+
extended_header_too_small
|
|
153
|
+
error
|
|
154
|
+
The file contains fewer extended-header bytes than expected.
|
|
155
|
+
data_block_too_small
|
|
156
|
+
error
|
|
157
|
+
The file contains fewer data-block bytes than expected.
|
|
158
|
+
data_block_exceeds_read_limit
|
|
159
|
+
error
|
|
160
|
+
The expected data block exceeds the configured read limit.
|
|
161
|
+
mrc_header_too_small
|
|
162
|
+
error
|
|
163
|
+
The file does not contain enough bytes for a full MRC header.
|
|
164
|
+
exception_during_validation
|
|
165
|
+
error
|
|
166
|
+
A Python exception-like message was emitted during validation.
|
|
167
|
+
unclassified_mrcfile_validation_message
|
|
168
|
+
warning
|
|
169
|
+
Fallback for any unrecognised message. The original message is
|
|
170
|
+
still preserved so new mrcfile messages are not silently lost.
|
|
171
|
+
Notes
|
|
172
|
+
-----
|
|
173
|
+
Some messages that ``mrcfile.validate()`` treats as validation failures
|
|
174
|
+
are intentionally classified as ``warning`` here, for example
|
|
175
|
+
``map_id_incorrect`` and ``mrc_format_version_invalid``. This lets VA
|
|
176
|
+
distinguish strict mrcfile validity from VA's own pipeline-level
|
|
177
|
+
acceptability.
|
|
178
|
+
Unknown messages are kept as
|
|
179
|
+
``unclassified_mrcfile_validation_message`` instead of being discarded.
|
|
180
|
+
This makes the parser forward-compatible with future mrcfile output.
|
|
181
|
+
"""
|
|
182
|
+
|
|
69
183
|
line = line.strip()
|
|
70
184
|
|
|
71
185
|
# 1. MRC map ID/header map field
|
|
@@ -552,6 +666,9 @@ def _parse_messages_and_warnings(messages_text, warning_messages):
|
|
|
552
666
|
|
|
553
667
|
|
|
554
668
|
def validate_single_map(map_input):
|
|
669
|
+
"""
|
|
670
|
+
Validate a single map file using mrcfile.validate() and capture any warnings.
|
|
671
|
+
"""
|
|
555
672
|
map_path = Path(_normalise_map_input(map_input))
|
|
556
673
|
messages = io.StringIO()
|
|
557
674
|
warning_messages = []
|
|
@@ -559,7 +676,7 @@ def validate_single_map(map_input):
|
|
|
559
676
|
result = OrderedDict([
|
|
560
677
|
("file", str(map_path)),
|
|
561
678
|
("exists", map_path.exists()),
|
|
562
|
-
("
|
|
679
|
+
("no_known_critical_error", False),
|
|
563
680
|
|
|
564
681
|
# New structured fields
|
|
565
682
|
("issue_count", 0),
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
"""Create RELION masks and FSC inputs from map volumes."""
|
|
2
|
+
|
|
1
3
|
from va.utils.ChimeraxViews import *
|
|
2
4
|
from va.utils.misc import find_rawmap_file
|
|
3
5
|
import numpy as np
|
|
@@ -57,6 +59,14 @@ def check_mrc(input_map):
|
|
|
57
59
|
|
|
58
60
|
@profile_peak_memory()
|
|
59
61
|
def relion_mask(raw_map, out_dir, mapname=None):
|
|
62
|
+
"""Create a RELION mask and a locally generated comparison mask.
|
|
63
|
+
|
|
64
|
+
The function invokes the RELION mask executable when available and writes
|
|
65
|
+
the resulting mask into ``out_dir``.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
str or None: Generated mask path when successful.
|
|
69
|
+
"""
|
|
60
70
|
relion_mask_executable_name = 'relion_mask_create'
|
|
61
71
|
relion_mask_executable = find_executable(relion_mask_executable_name)
|
|
62
72
|
if relion_mask_executable:
|
|
@@ -140,7 +150,9 @@ def relion_fsc(mapone, maptwo, mask_file=None, out_dir=None):
|
|
|
140
150
|
def relion_fsc_calculation(mapone, maptwo, root, mapname=None):
|
|
141
151
|
"""
|
|
142
152
|
Calculates FSc using Relion
|
|
143
|
-
|
|
153
|
+
|
|
154
|
+
Returns:
|
|
155
|
+
dict: Data dictionary containing the FSC result.
|
|
144
156
|
"""
|
|
145
157
|
|
|
146
158
|
result_type = 'fsc'
|
|
@@ -166,6 +178,7 @@ def relion_fsc_calculation(mapone, maptwo, root, mapname=None):
|
|
|
166
178
|
return None
|
|
167
179
|
|
|
168
180
|
def get_voxel_size(input_map):
|
|
181
|
+
"""Return the x-axis voxel size read from an MRC map."""
|
|
169
182
|
|
|
170
183
|
try:
|
|
171
184
|
with mrcfile.open(input_map, permissive=True) as mrc:
|
|
@@ -193,6 +206,18 @@ def calculate_pixels(angpix):
|
|
|
193
206
|
|
|
194
207
|
# following is the Python version of Relion auto masking
|
|
195
208
|
def relion_auto_mask_fast(data, threshold, extend_pixels, edge_width, verbose=True):
|
|
209
|
+
"""Build a soft-edged binary mask from an in-memory volume array.
|
|
210
|
+
|
|
211
|
+
Args:
|
|
212
|
+
data: Volume array to threshold.
|
|
213
|
+
threshold: Initial density threshold.
|
|
214
|
+
extend_pixels: Number of pixels to extend or shrink the mask.
|
|
215
|
+
edge_width: Width of the soft edge in pixels.
|
|
216
|
+
verbose: Whether to display progress messages.
|
|
217
|
+
|
|
218
|
+
Returns:
|
|
219
|
+
numpy.ndarray: Floating-point mask array.
|
|
220
|
+
"""
|
|
196
221
|
steps = []
|
|
197
222
|
if verbose:
|
|
198
223
|
steps = tqdm(total=3, desc="Masking Steps", unit="step")
|
|
@@ -239,6 +264,10 @@ def relion_auto_mask_fast(data, threshold, extend_pixels, edge_width, verbose=Tr
|
|
|
239
264
|
|
|
240
265
|
@profile_peak_memory()
|
|
241
266
|
def relion_auto_mask(input_file, output_file, threshold, extend_pixels, edge_width, verbose=True):
|
|
267
|
+
"""Create an MRC mask file from an input MRC volume.
|
|
268
|
+
|
|
269
|
+
The input voxel size is copied to the output file.
|
|
270
|
+
"""
|
|
242
271
|
with mrcfile.open(input_file) as mrc:
|
|
243
272
|
data = mrc.data.copy()
|
|
244
273
|
voxel_size = mrc.voxel_size.copy()
|
|
@@ -251,4 +280,3 @@ def relion_auto_mask(input_file, output_file, threshold, extend_pixels, edge_wid
|
|
|
251
280
|
|
|
252
281
|
if verbose:
|
|
253
282
|
print(f"== Mask saved to {output_file} ==")
|
|
254
|
-
|