emdbva 0.0.1.dev1__tar.gz → 0.0.1.dev2__tar.gz

{emdbva-0.0.1.dev1 → emdbva-0.0.1.dev2}/MANIFEST.in

@@ -7,6 +7,7 @@ exclude va/PATHS_loc.py
 exclude va/PATHS_codon.py
 exclude va/prepareandrun.py
 exclude va/prepareandrun_codon.py
+exclude va/prepareandrun_codon_slurm.py
 exclude va/test.log
 exclude va/vtkrender_nomodel.py
 exclude va/test/*

{emdbva-0.0.1.dev1/emdbva.egg-info → emdbva-0.0.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev1
+Version: 0.0.1.dev2
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang
@@ -26,6 +26,7 @@ Requires-Dist: matplotlib
 Requires-Dist: pandas
 Requires-Dist: biopython
 Requires-Dist: mrcfile
+Requires-Dist: torch
 Requires-Dist: opencv-python
 
 CryoEM validation analysis pipeline

{emdbva-0.0.1.dev1 → emdbva-0.0.1.dev2/emdbva.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev1
+Version: 0.0.1.dev2
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang
@@ -26,6 +26,7 @@ Requires-Dist: matplotlib
 Requires-Dist: pandas
 Requires-Dist: biopython
 Requires-Dist: mrcfile
+Requires-Dist: torch
 Requires-Dist: opencv-python
 
 CryoEM validation analysis pipeline

{emdbva-0.0.1.dev1 → emdbva-0.0.1.dev2}/emdbva.egg-info/SOURCES.txt

@@ -16,11 +16,13 @@ va/validationanalysis.py
 va/version.py
 va/metrics/__init__.py
 va/metrics/bars.py
+va/metrics/contour_level_predicator.py
 va/metrics/emda_mmcc.py
 va/metrics/emringer.py
 va/metrics/phaserandomization.py
 va/metrics/phenix_cc.py
 va/metrics/phenix_mm.py
+va/metrics/projections.py
 va/metrics/qscore.py
 va/metrics/resmap.py
 va/metrics/smoc.py

{emdbva-0.0.1.dev1 → emdbva-0.0.1.dev2}/emdbva.egg-info/requires.txt

@@ -7,4 +7,5 @@ matplotlib
 pandas
 biopython
 mrcfile
+torch
 opencv-python

{emdbva-0.0.1.dev1 → emdbva-0.0.1.dev2}/setup.py

@@ -48,7 +48,7 @@ setup(
         "pandas",
         "biopython",
         "mrcfile",
-        # "emda>=1.1.3.post3",
+        "torch",
         cv,
     ],
     classifiers=[

{emdbva-0.0.1.dev1 → emdbva-0.0.1.dev2}/va/mainva.py

@@ -53,6 +53,9 @@ def allruns(validationobj, runs):
     """
 
     # Projections
+    # if 'projection':
+    #     validationobj.new_projection()
+
     if 'projection' in runs:
         validationobj.orthogonal_projections()
         validationobj.rawmap_projections()
@@ -65,6 +68,10 @@ def allruns(validationobj, runs):
         validationobj.orthogonal_std()
         validationobj.rawmap_std()
 
+    # Predicated contour
+    if 'predictcontour' in runs:
+        validationobj.cl_prediction()
+
     # Central slice
     if 'central' in runs:
         validationobj.central_slice()

emdbva-0.0.1.dev2/va/metrics/contour_level_predicator.py

@@ -0,0 +1,157 @@
+import os
+os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
+import torch
+import torch.nn as nn
+import numpy as np
+torch.manual_seed(42)
+import warnings
+import mrcfile
+
+warnings.filterwarnings("ignore", message="CUDA initialization: The NVIDIA driver on your system is too old.*")
+warnings.filterwarnings("ignore", category=RuntimeWarning)
+warnings.filterwarnings("ignore", message="User provided device_type of 'cuda', but CUDA is not available. Disabling")
+
+
+# Convolutional layers adapted from https://www.nature.com/articles/s41598-022-19212-6
+class CNNModel(nn.Module):
+    def __init__(self):
+        super(CNNModel, self).__init__()
+        self.conv = nn.Sequential(
+            nn.Conv3d(1, 32, 7, 1, padding_mode='replicate'),
+            nn.BatchNorm3d(32),
+            nn.ReLU(),
+            nn.Conv3d(32, 64, 5, 1, padding_mode='replicate'),
+            nn.BatchNorm3d(64),
+            nn.ReLU(),
+            nn.MaxPool3d(2),
+            nn.Conv3d(64, 128, 5, 1, padding_mode='replicate'),
+            nn.BatchNorm3d(128),
+            nn.ReLU(),
+            nn.MaxPool3d(2),
+            nn.Conv3d(128, 256, 3, 1, padding_mode='replicate'),
+            nn.BatchNorm3d(256),
+            nn.ReLU(),
+            nn.MaxPool3d(2),
+            nn.Conv3d(256, 384, 1, 1, padding_mode='replicate'),
+            nn.ReLU(),
+            nn.AvgPool3d(3, stride=2),
+            nn.Dropout(0.3))
+
+        self.mlp = nn.Sequential(nn.Linear(in_features=384, out_features=64),
+                                 nn.ReLU(),
+                                 nn.Linear(in_features=64, out_features=1))
+
+    def forward(self, x):
+        x = x.view(-1, 1, 64, 64, 64)
+        x = self.conv(x)
+        x = x.view(x.size(0), -1)
+        x = self.mlp(x)
+        return x
+
+
+def check_boxsize(vol):
+    return vol.shape[0]
+
+
+def load_data(vol_d_norm):
+    vol_array = np.empty(shape=(1, 64, 64, 64), dtype=np.float32)
+    vol_array[:] = vol_d_norm
+    data_generator = torch.utils.data.DataLoader(vol_array, batch_size=1, shuffle=False)
+    return data_generator
+
+
+def downsample(vol):
+    boxsize = check_boxsize(vol)
+    vol_ft = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(vol)))
+    start = boxsize // 2 - 64 // 2
+    stop = boxsize // 2 + 64 // 2
+    vol_ft_d = vol_ft[start:stop, start:stop, start:stop]
+    vol_d = np.fft.ifftshift(np.fft.ifftn(np.fft.ifftshift(vol_ft_d)))
+    vol_d = vol_d.real
+    vol_d *= ((64 / boxsize) ** 3)
+    vol_d = vol_d.astype(np.float32)
+    return vol_d
+
+
+def upsample(
+        vol):  # through interpolation of maps in Fourier space; based on https://www.sciencedirect.com/science/article/pii/S001046551400335X
+    boxsize = check_boxsize(vol)
+    new_box_size = 64
+    vol_ft = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(vol)))
+    padded_vol_ft = np.zeros((new_box_size, new_box_size, new_box_size), dtype=np.float32)
+    padded_vol_ft[:vol.shape[0], :vol.shape[1], :vol.shape[2]] = vol_ft
+    interpolated_density_map = np.fft.ifftshift(np.fft.ifftn(np.fft.ifftshift((padded_vol_ft))))
+    interpolated_density_map = interpolated_density_map.real
+    interpolated_density_map *= ((64 / boxsize) ** 3)
+    return interpolated_density_map
+
+
+def normalize(vol_data):
+    d_upper = np.percentile(vol_data, 99.999)
+    d_lower = np.percentile(vol_data, 0.001)
+    vol_data = np.where(vol_data > d_upper, d_upper, vol_data)
+    vol_data = np.where(vol_data < d_lower, d_lower, vol_data)
+    vol_d_norm = np.where(vol_data <= 0, 0, vol_data)
+    vol_d_min = np.min(vol_data[vol_data > 0])
+    vol_d_max = np.max(vol_d_norm)
+    vol_d_norm = (vol_d_norm - vol_d_min) / (vol_d_max - vol_d_min)
+    vol_d_norm = 2 * vol_d_norm - 1
+    return vol_d_norm, vol_d_min, vol_d_max
+
+
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device('cpu')
+model = CNNModel()
+dir = os.path.dirname(os.path.abspath(__file__))
+parent_dir = os.path.dirname(dir)
+weights_dir = os.path.join(parent_dir, 'utils')
+checkpoint = torch.load(os.path.join(weights_dir, 'cl_weights.pth'), map_location=device)
+model.load_state_dict(checkpoint['model_state_dict'])
+
+
+def model_pred(data_generator):
+    model.eval()
+    pred_list = []
+    with torch.no_grad():
+        with torch.cuda.amp.autocast():
+            for (idx, x) in enumerate(data_generator):
+                x = x.to(device)
+                prediction = model(x)
+                prediction = (prediction.squeeze(dim=1))
+                pred_list.append(prediction)
+    return pred_list
+
+
+def calc_level_dev(vol_data):
+    boxsize = check_boxsize(vol_data)
+
+    if boxsize > 64:
+        _, vol_min, vol_max = normalize(vol_data)
+        vol_d = downsample(vol_data)
+        vol_d_norm, _, _ = normalize(vol_d)
+
+    elif boxsize < 64:
+        _, vol_min, vol_max = normalize(vol_data)
+        vol_d = upsample(vol_data)
+        vol_d_norm, _, _ = normalize(vol_d)
+
+    else:
+        vol_d = vol_data
+        vol_d_norm, vol_d_min, vol_d_max = normalize(vol_d)
+
+    data_generator = load_data(vol_d_norm)
+    pred_list = model_pred(data_generator)
+    all_preds = np.concatenate(pred_list, axis=0)
+    int_preds = (all_preds + 1) / 2
+
+    if boxsize != 64:
+        norm_preds = (int_preds * (vol_max - vol_min)) + vol_min
+    else:
+        norm_preds = (int_preds * (vol_d_max - vol_d_min)) + vol_d_min
+
+    return norm_preds
+
+
+
+
+
+

emdbva-0.0.1.dev2/va/metrics/projections.py

@@ -0,0 +1,355 @@
+import os
+import sys
+import timeit
+import json
+import codecs
+import cv2
+import numpy as np
+from math import ceil
+from scipy import ndimage
+from mrcfile.mrcfile import MrcFile
+import inspect
+from va.utils.misc import out_json
+
+class Projections:
+    def __init__(self, primary_map, rawmap=None, workdir=None, platform=None):
+        self.map = primary_map
+        self.rawmap = rawmap
+        self.workdir = workdir
+        self.platform = platform
+        self.errlist = []
+
+    def mapincheck(self, mapin, workdirin):
+        frame = inspect.currentframe()
+        func = inspect.getframeinfo(frame.f_back).function
+        map, workdir = None, None
+
+        if mapin is not None:
+            if isinstance(mapin, str):
+                if os.path.isfile(mapin):
+                    map = MrcFile(mapin, mode='r')
+                else:
+                    print('Map does not exist.')
+            elif isinstance(mapin, MrcFile):
+                map = mapin
+            else:
+                print(f'Function:{func} only accepts a string of the full map name or a TEMPy Map object as input.')
+
+        if workdirin is not None:
+            if isinstance(workdirin, str):
+                if os.path.isdir(workdirin):
+                    workdir = workdirin
+                else:
+                    print('Output directory does not exist.')
+            else:
+                print(f'Function:{func} only accepts a string as the directory parameter.')
+
+        return map, workdir
+
+    def resize_img(self, input_arr, output_img):
+        height, width = input_arr.shape[0:2]
+        if width >= height:
+            largerscaler = 300. / width
+            newheight = int(ceil(largerscaler * height))
+            resized_img = cv2.resize(input_arr, (300, newheight), interpolation=cv2.INTER_LANCZOS4)
+        else:
+            largerscaler = 300. / height
+            newwidth = int(ceil(largerscaler * width))
+            resized_img = cv2.resize(input_arr, (newwidth, 300), interpolation=cv2.INTER_LANCZOS4)
+
+        cv2.imwrite(output_img, resized_img)
+
+    def scale_img(self, projection):
+        onevalue = True if projection.min() == projection.max() else False
+        if onevalue:
+            rescaled = np.full((projection.shape), 0.0).astype('uint8')
+        else:
+            rescaled = (((projection - projection.min()) * 255.0) / (projection.max() - projection.min())).astype('uint8')
+        return rescaled
+
+    def get_projection(self, map, type, axis):
+        mapdata = map.data
+        ind = None
+        if type == 'projection':
+            projection = np.sum(mapdata, axis=axis)
+        elif type == 'max':
+            projection = np.amax(mapdata, axis=axis)
+        elif type == 'min':
+            projection = np.amin(mapdata, axis=axis)
+        elif type == 'std':
+            projection = np.std(mapdata, axis=axis)
+        elif type == 'median':
+            projection = np.median(mapdata, axis=axis)
+        elif type == 'central':
+            projection = map.data[:, :, int(map.header.nx / 2)] if axis == 2 else \
+                         map.data[:, int(map.header.ny / 2), :] if axis == 1 else \
+                         map.data[int(map.header.nz / 2), :, :] if axis == 0 else None
+            if axis == 2:
+                ind = {'x': int(map.header.nx / 2)}
+            elif axis == 1:
+                ind = {'y': int(map.header.ny / 2)}
+            elif axis == 0:
+                ind = {'z': int(map.header.nz / 2)}
+            else:
+                ind = None
+
+        elif type == 'largestvariance':
+            if axis == 2:
+                xvar = [ndimage.variance(map.data[:, :, i]) for i in range(map.data.shape[2])]
+                xlargeind = int(np.argmax(xvar))
+                ind = {'x': xlargeind}
+                projection = map.data[:, :, xlargeind]
+            elif axis == 1:
+                yvar = [ndimage.variance(map.data[:, i, :]) for i in range(map.data.shape[1])]
+                ylargeind = int(np.argmax(yvar))
+                ind = {'y': ylargeind}
+                projection = map.data[:, ylargeind, :]
+            elif axis == 0:
+                zvar = [ndimage.variance(map.data[i, :, :]) for i in range(map.data.shape[0])]
+                zlargeind = int(np.argmax(zvar))
+                ind = {'z': zlargeind}
+                projection = map.data[zlargeind, :, :]
+            else:
+                projection = None
+        else:
+            projection = None
+            print('Wrong type. Use one of: projection, max, min, std, median, central, largestvariance.')
+        return projection, ind
+
+
+    def output_filename(self, axis_letter, mapname, type, proc_fun=None):
+        if proc_fun:
+            suffix = 'glow_' if proc_fun == self.glowimage else ''
+        else:
+            suffix = ''
+
+        if type == 'central' or type == 'largestvariance':
+            output_name = f'{self.workdir}{mapname}_{axis_letter}{type}_slice.jpeg'
+            scaled_output_name = f'{self.workdir}{mapname}_scaled_{axis_letter}{type}_slice.jpeg'
+        else:
+            output_name = f'{self.workdir}{mapname}_{suffix}{axis_letter}{type}.jpeg'
+            scaled_output_name = f'{self.workdir}{mapname}_scaled_{suffix}{axis_letter}{type}.jpeg'
+
+        return output_name, scaled_output_name
+
+    def save_scale(self, reposition, output_name, scaled_output_name, errlist):
+
+        try:
+            cv2.imwrite(output_name, reposition)
+        except IOError as ioerr:
+            err = f'Saving {output_name} projection err: {ioerr}'
+            errlist.append(err)
+            sys.stderr.write(err + '\n')
+
+        try:
+            self.resize_img(reposition, scaled_output_name)
+        except Exception as e:
+            err = f'Saving scaled {scaled_output_name} image err: {e}'
+            errlist.append(err)
+            sys.stderr.write(err + '\n')
+
+    def map_to_img(self, map, axis, type, errlist, proc_fun=None):
+        projection, ind = self.get_projection(map, type, axis)
+        org = {}
+        scale = {}
+        if projection is not None:
+            rescaled = self.scale_img(projection)
+            if axis == 2:
+                reposition = np.flipud(rescaled)
+                axis_letter = 'x'
+            elif axis == 1:
+                reposition = np.rot90(rescaled)
+                axis_letter = 'y'
+            elif axis == 0:
+                reposition = np.flipud(rescaled)
+                axis_letter = 'z'
+            else:
+                return None
+
+            mapname = os.path.basename(map.fullname)
+            output_name, scaled_output_name = self.output_filename(axis_letter, mapname, type, proc_fun)
+            org[axis_letter] = os.path.basename(output_name)
+            scale[axis_letter] = os.path.basename(scaled_output_name)
+
+            self.save_scale(reposition, output_name, scaled_output_name, errlist)
+            if proc_fun:
+                glowimage = proc_fun(reposition)
+                self.save_scale(glowimage, output_name, scaled_output_name, errlist)
+
+        return ind, org, scale
+
+    def orthogonal_projections(self, mapin=None, workdir=None, type=None, label=''):
+        map, workdir = self.mapincheck(mapin, workdir)
+        if map is not None and workdir is not None:
+            start = timeit.default_timer()
+            self.errlist = []
+            mapname = os.path.basename(map.fullname)
+            if not type:
+                types = ['projection', 'max', 'min', 'std', 'median', 'central', 'largestvariance']
+            else:
+                types = [type]
+            result_dict = {}
+            for type in types:
+                ind_result = {}
+                org_result = {}
+                scale_result = {}
+                final_org = {}
+                final_scale = {}
+                final_ind = {}
+                glow_ind_result = {}
+                glow_org_result = {}
+                glow_scale_result = {}
+                for axis in range(2, -1, -1):
+                    ind, org, scale = self.map_to_img(map, axis, type, self.errlist)
+                    if type != 'max' or type == 'projection' or type == 'std':
+                        glow_ind, glow_org, glow_scale = self.map_to_img(map, axis, type, self.errlist, self.glowimage)
+                        glow_org_result.update(glow_org)
+                        glow_scale_result.update(glow_scale)
+                        if type == 'central' or type == 'largestvariance':
+                            glow_ind_result.update(glow_ind)
+                    if type == 'central' or type == 'largestvariance':
+                        ind_result.update(ind)
+                    org_result.update(org)
+                    scale_result.update(scale)
+                final_org['original'] = org_result
+                final_scale['scaled'] = scale_result
+                if type == 'central' or type == 'largestvariance':
+                    final_ind['indices'] = ind_result
+
+                if type == 'largestvariance':
+                    result_dict['largest_variance_slice'] = {**final_org, **final_scale, **final_ind}
+                elif type == 'central':
+                    result_dict['central_slice'] = {**final_org, **final_scale, **final_ind}
+                else:
+                    result_dict[f'orthogonal_{type}'] = {**final_org, **final_scale}
+
+            if len(types) == 1:
+                json_file = f'{self.workdir}{mapname}_{type}.json'
+            else:
+                if not label:
+                    json_file = f'{self.workdir}{mapname}_primary_projections.json'
+                else:
+                    json_file = f'{self.workdir}{mapname}_raw_projections.json'
+
+            out_json(result_dict, json_file)
+            end = timeit.default_timer()
+            print(f'Projections and their glow projections time for {mapname}: {end - start}')
+            print('------------------------------------')
+        else:
+            print('No orthogonal projections without proper map input and the output directory information.')
+
+    def rawmap_projections(self):
+        if self.rawmap is not None:
+            self.orthogonal_projections(self.rawmap, self.workdir, 'rawmap_')
+        else:
+            print('No raw map to calculate orthogonal projections.')
+
+
+    def save_to_json(self, errlist, mapname, workdir, label):
+        bothjson = {}
+        projectionjson = {}
+        orgprojectionjson = {}
+
+        for axis_letter in ['x', 'y', 'z']:
+            projectionjson[axis_letter] = {}
+            orgprojectionjson[axis_letter] = {}
+
+        for axis in range(3):
+            for type in ['projection', 'max', 'min', 'std', 'median', 'central', 'largestvariance']:
+                for axis_letter in ['x', 'y', 'z']:
+                    projection_name = f'{workdir}{mapname}_{axis_letter}{type}.jpeg'
+                    scaled_projection_name = f'{workdir}{mapname}_scaled_{axis_letter}{type}.jpeg'
+                    projectionjson[axis_letter][type] = os.path.basename(projection_name) if os.path.isfile(projection_name) else None
+                    orgprojectionjson[axis_letter][type] = os.path.basename(scaled_projection_name) if os.path.isfile(scaled_projection_name) else None
+
+        bothjson['original'] = orgprojectionjson
+        bothjson['scaled'] = projectionjson
+
+        if errlist:
+            bothjson['err'] = {'orthogonal_median_err': errlist}
+        finaldict = {label + 'orthogonal_median': bothjson}
+
+        try:
+            with codecs.open(f'{workdir}{mapname}_median.json', 'w', encoding='utf-8') as f:
+                json.dump(finaldict, f)
+        except IOError as ioerr:
+            jsonerr = f'Saving median projection into json error: {ioerr}'
+            sys.stderr.write(jsonerr + '\n')
+
+    def glowimage(self, im_gray):
+        """Applies a glow color map using cv2.applyColorMap()"""
+
+        # Create the LUT:
+        lut = np.zeros((256, 1, 3), dtype=np.uint8)
+        lut[:, 0, 2] = [0, 1, 1, 2, 2, 3, 4, 6, 8, 11, 13, 15, 18,
+                        21, 23, 24, 27, 30, 31, 33, 36, 37, 40, 42, 45, 46,
+                        49, 51, 53, 56, 58, 60, 62, 65, 68, 70, 72, 74, 78,
+                        80, 82, 84, 86, 89, 91, 93, 96, 98, 100, 102, 104, 106,
+                        108, 110, 113, 115, 117, 119, 122, 125, 127, 129, 132, 135, 135,
+                        137, 140, 141, 142, 145, 148, 149, 152, 154, 156, 157, 158, 160,
+                        162, 164, 166, 168, 170, 171, 173, 174, 176, 178, 179, 180, 182,
+                        183, 185, 186, 189, 192, 193, 193, 194, 195, 195, 196, 198, 199,
+                        201, 203, 204, 204, 205, 206, 207, 209, 211, 211, 211, 211, 213,
+                        215, 216, 216, 216, 216, 218, 219, 219, 219, 220, 222, 223, 223,
+                        223, 223, 224, 224, 226, 227, 227, 227, 227, 228, 229, 231, 231,
+                        231, 231, 231, 231, 231, 232, 233, 234, 234, 234, 234, 234, 234,
+                        235, 237, 238, 238, 238, 238, 238, 238, 238, 238, 239, 240, 242,
+                        242, 242, 242, 242, 242, 242, 242, 242, 243, 245, 246, 246, 246,
+                        246, 245, 245, 245, 245, 245, 245, 245, 247, 248, 249, 249, 249,
+                        249, 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, 249,
+                        249, 250, 252, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253,
+                        253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253,
+                        253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253,
+                        253, 253, 253, 253, 253, 253, 253, 253, 0]
+
+        lut[:, 0, 1] = [138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                        1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2,
+                        2, 2, 2, 2, 3, 3, 3, 3, 4, 5, 5, 5, 5,
+                        6, 6, 6, 7, 7, 8, 9, 9, 9, 9, 10, 10, 11,
+                        12, 13, 14, 14, 14, 14, 15, 16, 16, 17, 18, 19, 19,
+                        19, 20, 21, 22, 23, 24, 24, 25, 27, 28, 28, 28, 29,
+                        31, 32, 32, 33, 35, 36, 36, 37, 39, 40, 40, 41, 42,
+                        43, 45, 46, 47, 48, 50, 51, 52, 54, 55, 56, 57, 59,
+                        62, 63, 65, 66, 68, 70, 71, 73, 74, 75, 77, 79, 81,
+                        83, 85, 87, 89, 92, 93, 96, 98, 99, 100, 103, 105, 107,
+                        109, 111, 113, 116, 117, 119, 121, 123, 125, 126, 128, 130, 132,
+                        135, 137, 139, 140, 142, 144, 145, 147, 148, 150, 152, 154, 156,
+                        158, 160, 161, 162, 164, 166, 168, 171, 172, 172, 174, 176, 177,
+                        179, 180, 182, 183, 185, 187, 188, 189, 191, 192, 192, 192, 196,
+                        200, 201, 201, 202, 203, 204, 206, 208, 210, 212, 213, 213, 214,
+                        215, 217, 218, 220, 221, 222, 223, 224, 225, 226, 226, 227, 228,
+                        229, 231, 232, 233, 234, 235, 236, 237, 238, 239, 239, 239, 239,
+                        241, 242, 243, 243, 243, 244, 245, 247, 247, 247, 247, 247, 247,
+                        248, 249, 250, 251, 251, 251, 252, 252, 252, 252, 252, 252, 252,
+                        252, 252, 252, 252, 252, 252, 253, 253, 0]
+
+        lut[:, 0, 0] = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 4, 4,
+                        4, 4, 4, 4, 3, 3, 3, 3, 4, 6, 6, 6, 6,
+                        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
+                        6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 7, 9, 10,
+                        10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
+                        10, 10, 10, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+                        9, 9, 9, 9, 8, 8, 8, 8, 8, 8, 8, 8, 8,
+                        8, 8, 8, 8, 8, 8, 8, 8, 7, 7, 7, 7, 7,
+                        7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6,
+                        6, 6, 6, 6, 7, 10, 12, 12, 12, 12, 12, 12, 13,
+                        15, 17, 18, 18, 19, 20, 22, 23, 23, 24, 26, 27, 27,
+                        28, 30, 32, 33, 34, 35, 37, 39, 40, 41, 43, 45, 46,
+                        48, 50, 52, 54, 55, 57, 58, 61, 63, 65, 67, 70, 72,
+                        73, 76, 79, 80, 83, 86, 88, 89, 92, 95, 96, 99, 102,
+                        104, 107, 108, 110, 113, 116, 120, 121, 123, 125, 127, 129, 132,
+                        135, 137, 140, 143, 146, 149, 150, 153, 155, 158, 160, 162, 165,
+                        168, 171, 173, 175, 178, 180, 183, 186, 188, 191, 192, 195, 198,
+                        201, 203, 206, 209, 210, 213, 215, 218, 221, 223, 225, 228, 231,
+                        233, 236, 238, 241, 244, 246, 250, 252, 255]
+
+        im_color = cv2.applyColorMap(im_gray, lut)
+        return im_color
+
+# Example usage:
+# rawmap = MrcFile("/Users/zhe/Downloads/tests/VA/nfs/msd/work2/emdb/development/staging/em/81/8117/va/emd_8117_rawmap.map")
+# workdir = "/Users/zhe/Downloads/tests/VA/nfs/msd/work2/emdb/development/staging/em/81/8117/va"
+# op = Projections(rawmap, workdir)
+# op.rawmap_projections()
+# op.rawmap_median()

{emdbva-0.0.1.dev1 → emdbva-0.0.1.dev2}/va/preparation.py

@@ -327,7 +327,7 @@ class PreParation:
                        'volume': False, 'fsc': False, 'raps': False, 'mapmodel': False, 'inclusion': False,
                        'largestvariance': False, 'mask': False, 'symmetry': False, 'rmmcc': False, 'smoc': False,
                        'resccc': False, 'emringer': False, 'strudel': False, '3dfsc': False, 'locres': False,
-                       'phrand': False}
+                       'phrand': False, 'predictcontour': False}
             for key in resdict.keys():
                 if key in runs:
                     resdict[key] = True
@@ -341,7 +341,7 @@ class PreParation:
             if self.mapname is not None:
                 runlist.extend(['projection', 'central', 'surface', 'volume', 'density', 'raps', 'largestvariance',
                                 'mask', 'fsc', 'mmfsc', 'rmmcc', 'symmetry', 'qscore', 'strudel', 'emringer', '3dfsc',
-                                'smoc', 'resccc', 'locres', 'phrand'
+                                'smoc', 'resccc', 'locres', 'phrand', 'predictcontour',
                                 ])
 
             if self.masks is None:
@@ -705,119 +705,6 @@ class PreParation:
             return None
 
 
-    def swap_axes(self, header):
-        """
-
-            Swap the axes to make the data arranged as z, y, z by indices
-
-        :param header:
-        :return:
-        """
-        pass
-
-
-    # def frommrc_totempy(self, fullmapname):
-    #     """
-    #         Transfer the mrcfile map object to TEMPy map object
-    #
-    #     :param fullmapname:
-    #     :return: TEMPy map object
-    #     """
-    #
-    #     mrcmap = mrcfile.mmap(fullmapname, mode='r+')
-    #     mrcheader = mrcmap.header
-    #     # mapdata = mrcmap.data.astype('float')
-    #     mapdata = mrcmap.data
-    #     crs = (mrcheader.mapc, mrcheader.mapr, mrcheader.maps)
-    #     print('----original start------')
-    #     print(mrcheader)
-    #     print(mapdata.shape)
-    #     print('------------------')
-    #     reversecrs = crs[::-1]
-    #     nstarts = (mrcheader.nxstart, mrcheader.nystart, mrcheader.nzstart)
-    #     stdcrs = (3, 2, 1)
-    #     diffcrs = tuple(x-y for x, y in zip(reversecrs, stdcrs))
-    #     if diffcrs != (0, 0, 0):
-    #         if 1 in diffcrs and -1 in diffcrs:
-    #             mapdata = np.swapaxes(mapdata, diffcrs.index(-1), diffcrs.index(1))
-    #         if -2 in diffcrs and 2 in diffcrs:
-    #             mapdata = np.swapaxes(mapdata, diffcrs.index(-2), diffcrs.index(2))
-    #         if 1 in diffcrs and -2 in diffcrs:
-    #             mapdata = np.swapaxes(np.swapaxes(mapdata, 0, 1), 1, 2)
-    #         if -1 in diffcrs and 2 in diffcrs:
-    #             mapdata = np.swapaxes(np.swapaxes(mapdata, 1, 2), 0, 1)
-    #         # mapdata = np.swapaxes(mapdata, 0, 2)
-    #
-    #     print(mapdata.shape)
-    #     # mrcmap.set_data(mapdata)
-    #     # mrcmap.update_header_from_data()
-    #     # mapdata = np.swapaxes(np.swapaxes(mapdata, 0, 2), 1, 2)
-    #     tempyheader = MapParser.readMRCHeader(fullmapname)
-    #
-    #     # nx, ny, nz and nxstart, nystart, nzstart haver to be changed accordingly to the data
-    #     tempyheader = list(tempyheader)
-    #     tempyheader[0] = mapdata.shape[2]
-    #     tempyheader[1] = mapdata.shape[1]
-    #     tempyheader[2] = mapdata.shape[0]
-    #     tempyheader[3] = mrcheader.mode.item(0)
-    #     tempyheader[4] = nstarts[0].item(0)
-    #     tempyheader[5] = nstarts[1].item(0)
-    #     tempyheader[6] = nstarts[2].item(0)
-    #     tempyheader[7] = mrcheader.mx.item(0)
-    #     tempyheader[8] = mrcheader.my.item(0)
-    #     tempyheader[9] = mrcheader.mz.item(0)
-    #     tempyheader[10] = mrcheader.cella.x.item(0)
-    #     tempyheader[11] = mrcheader.cella.y.item(0)
-    #     tempyheader[12] = mrcheader.cella.z.item(0)
-    #     tempyheader[13:16] = mrcheader.cellb.tolist()
-    #     tempyheader[16] = crs[0].item(0)
-    #     tempyheader[17] = crs[1].item(0)
-    #     tempyheader[18] = crs[2].item(0)
-    #     tempyheader[19] = mrcheader.dmin.item(0)
-    #     tempyheader[20] = mrcheader.dmax.item(0)
-    #     tempyheader[21] = mrcheader.dmean.item(0)
-    #     tempyheader[22] = mrcheader.ispg.item(0)
-    #     tempyheader[23] = mrcheader.nsymbt.item(0)
-    #
-    #     # tempyheader[24] = mrcheader.extra1.item(0)
-    #     # tempyheader[25] = mrcheader.exttyp.item(0)
-    #     # tempyheader[26] = mrcheader.nversion.item(0)
-    #     # tempyheader[27] = mrcheader.extra2.item(0)
-    #
-    #     tempyheader[49] = mrcheader.origin.x.item(0)
-    #     tempyheader[50] = mrcheader.origin.y.item(0)
-    #     tempyheader[51] = mrcheader.origin.z.item(0)
-    #     tempyheader[52:55] = ['M', 'A', 'P']
-    #
-    #     tempyheader[56] = mrcheader.machst
-    #     tempyheader[57] = mrcheader.rms.item(0)
-    #     # tempyheader[58] = mrcheader.nlabl.item(0)
-    #     # tempyheader[59] = mrcheader.label.item(0)
-    #
-    #
-    #
-    #     # tempyheader[13] = mrcheader.cellb.x.item(0)
-    #     # tempyheader[14] = mrcheader.cellb.y.item(0)
-    #     # tempyheader[15] = mrcheader.cellb.z.item(0)
-    #     # here also keep the nstarts position according to original crs order as in __getindices it automaticlly
-    #     # adjust the calculation based on the
-    #     # tempyheader[4] = int(nstarts[crs.index(1)])
-    #     # tempyheader[5] = int(nstarts[crs.index(2)])
-    #     # tempyheader[6] = int(nstarts[crs.index(3)])
-    #     tempyheader = tuple(tempyheader)
-    #     origin = (float(mrcheader.origin.x), float(mrcheader.origin.y), float(mrcheader.origin.z))
-    #     apix = (mrcheader.cella.x/mrcheader.mx, mrcheader.cella.y/mrcheader.my, mrcheader.cella.z/mrcheader.mz)[0]
-    #
-    #     finalmap = Map(mapdata, origin, apix, fullmapname, header=tempyheader)
-    #     print(finalmap.box_size())
-    #     print('---box---')
-    #
-    #     # inputmap.fullMap = np.swapaxes(np.swapaxes(mrcfile.mmap(fullmapname).data, 0, 2), 1, 2)
-    #     # print(inputmap.fullMap.shape)
-    #     # print(inputmap.fullMap[1, 3, 66])
-    #
-    #     return finalmap
-
     # @profile
     def write_map(self, outmap_name, mapdata, nstarts, org_header):
         """
@@ -1557,7 +1444,14 @@ class PreParation:
             if key in obj1 and key in obj2 and isinstance(obj1[key], dict) and isinstance(obj2[key], dict):
                 merged_obj[key] = self.merge_json_objects(obj1[key], obj2[key])
             elif key in obj1 and key in obj2:
-                merged_obj[key] = {**obj1[key], **obj2[key]}
+                if isinstance(obj1[key], dict) and isinstance(obj2[key], dict):
+                    merged_obj[key] = {**obj1[key], **obj2[key]}
+                elif isinstance(obj1[key], dict) and not isinstance(obj2[key], dict):
+                    merged_obj[key] = obj1[key]
+                elif not isinstance(obj1[key], dict) and isinstance(obj2[key], dict):
+                    merged_obj[key] = obj2[key]
+                else:
+                    merged_obj[key] = obj1[key]
             elif key in obj1:
                 merged_obj[key] = obj1[key]
             elif key in obj2:

{emdbva-0.0.1.dev1 → emdbva-0.0.1.dev2}/va/validationanalysis.py

@@ -74,6 +74,8 @@ from va.metrics.bars import *
 from va.metrics.phaserandomization import *
 from va.utils.stars import *
 from va.utils.misc import *
+from va.metrics.contour_level_predicator import *
+from va.metrics.projections import *
 import va
 
 try:
@@ -295,71 +297,6 @@ class ValidationAnalysis:
                         201, 203, 206, 209, 210, 213, 215, 218, 221, 223, 225, 228, 231,
                         233, 236, 238, 241, 244, 246, 250, 252, 255]
 
-        # Modified Glow LUT which make the first four smallest value as green
-        # lut[:, 0, 2] = [0, 0, 0, 0, 2, 3, 4, 6, 8, 11, 13, 15, 18,
-        #                 21, 23, 24, 27, 30, 31, 33, 36, 37, 40, 42, 45, 46,
-        #                 49, 51, 53, 56, 58, 60, 62, 65, 68, 70, 72, 74, 78,
-        #                 80, 82, 84, 86, 89, 91, 93, 96, 98, 100, 102, 104, 106,
-        #                 108, 110, 113, 115, 117, 119, 122, 125, 127, 129, 132, 135, 135,
-        #                 137, 140, 141, 142, 145, 148, 149, 152, 154, 156, 157, 158, 160,
-        #                 162, 164, 166, 168, 170, 171, 173, 174, 176, 178, 179, 180, 182,
-        #                 183, 185, 186, 189, 192, 193, 193, 194, 195, 195, 196, 198, 199,
-        #                 201, 203, 204, 204, 205, 206, 207, 209, 211, 211, 211, 211, 213,
-        #                 215, 216, 216, 216, 216, 218, 219, 219, 219, 220, 222, 223, 223,
-        #                 223, 223, 224, 224, 226, 227, 227, 227, 227, 228, 229, 231, 231,
-        #                 231, 231, 231, 231, 231, 232, 233, 234, 234, 234, 234, 234, 234,
-        #                 235, 237, 238, 238, 238, 238, 238, 238, 238, 238, 239, 240, 242,
-        #                 242, 242, 242, 242, 242, 242, 242, 242, 243, 245, 246, 246, 246,
-        #                 246, 245, 245, 245, 245, 245, 245, 245, 247, 248, 249, 249, 249,
-        #                 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, 249,
-        #                 249, 250, 252, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253,
-        #                 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253,
-        #                 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253,
-        #                 253, 253, 253, 253, 253, 253, 253, 253, 0]
-        #
-        # lut[:, 0, 1] = [138, 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        #                 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2,
-        #                 2, 2, 2, 2, 3, 3, 3, 3, 4, 5, 5, 5, 5,
-        #                 6, 6, 6, 7, 7, 8, 9, 9, 9, 9, 10, 10, 11,
-        #                 12, 13, 14, 14, 14, 14, 15, 16, 16, 17, 18, 19, 19,
-        #                 19, 20, 21, 22, 23, 24, 24, 25, 27, 28, 28, 28, 29,
-        #                 31, 32, 32, 33, 35, 36, 36, 37, 39, 40, 40, 41, 42,
-        #                 43, 45, 46, 47, 48, 50, 51, 52, 54, 55, 56, 57, 59,
-        #                 62, 63, 65, 66, 68, 70, 71, 73, 74, 75, 77, 79, 81,
-        #                 83, 85, 87, 89, 92, 93, 96, 98, 99, 100, 103, 105, 107,
-        #                 109, 111, 113, 116, 117, 119, 121, 123, 125, 126, 128, 130, 132,
-        #                 135, 137, 139, 140, 142, 144, 145, 147, 148, 150, 152, 154, 156,
-        #                 158, 160, 161, 162, 164, 166, 168, 171, 172, 172, 174, 176, 177,
-        #                 179, 180, 182, 183, 185, 187, 188, 189, 191, 192, 192, 192, 196,
-        #                 200, 201, 201, 202, 203, 204, 206, 208, 210, 212, 213, 213, 214,
-        #                 215, 217, 218, 220, 221, 222, 223, 224, 225, 226, 226, 227, 228,
-        #                 229, 231, 232, 233, 234, 235, 236, 237, 238, 239, 239, 239, 239,
-        #                 241, 242, 243, 243, 243, 244, 245, 247, 247, 247, 247, 247, 247,
-        #                 248, 249, 250, 251, 251, 251, 252, 252, 252, 252, 252, 252, 252,
-        #                 252, 252, 252, 252, 252, 252, 253, 253, 0]
-        #
-        # lut[:, 0, 0] = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        #                 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 4, 4,
-        #                 4, 4, 4, 4, 3, 3, 3, 3, 4, 6, 6, 6, 6,
-        #                 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
-        #                 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 7, 9, 10,
-        #                 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
-        #                 10, 10, 10, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
-        #                 9, 9, 9, 9, 8, 8, 8, 8, 8, 8, 8, 8, 8,
-        #                 8, 8, 8, 8, 8, 8, 8, 8, 7, 7, 7, 7, 7,
-        #                 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6,
-        #                 6, 6, 6, 6, 7, 10, 12, 12, 12, 12, 12, 12, 13,
-        #                 15, 17, 18, 18, 19, 20, 22, 23, 23, 24, 26, 27, 27,
-        #                 28, 30, 32, 33, 34, 35, 37, 39, 40, 41, 43, 45, 46,
-        #                 48, 50, 52, 54, 55, 57, 58, 61, 63, 65, 67, 70, 72,
-        #                 73, 76, 79, 80, 83, 86, 88, 89, 92, 95, 96, 99, 102,
-        #                 104, 107, 108, 110, 113, 116, 120, 121, 123, 125, 127, 129, 132,
-        #                 135, 137, 140, 143, 146, 149, 150, 153, 155, 158, 160, 162, 165,
-        #                 168, 171, 173, 175, 178, 180, 183, 186, 188, 191, 192, 195, 198,
-        #                 201, 203, 206, 209, 210, 213, 215, 218, 221, 223, 225, 228, 231,
-        #                 233, 236, 238, 241, 244, 246, 250, 252, 255]
-
-        # Apply color map using cv2.applyColorMap()
         im_color = cv2.applyColorMap(im_gray, lut)
         return im_color
 
@@ -442,6 +379,13 @@ class ValidationAnalysis:
         im_color = cv2.applyColorMap(im_gray, lut)
         return im_color
 
+
+    def new_projection(self, type=None):
+        op = Projections(self.map, self.rawmap, self.workdir, self.platform)
+        op.orthogonal_projections(self.map, self.workdir, type)
+        if self.rawmap:
+            op.orthogonal_projections(self.rawmap, self.workdir, type, 'rawmap_')
+
     def mapincheck(self, mapin, workdirin):
 
         import inspect
@@ -7655,6 +7599,37 @@ class ValidationAnalysis:
         phaserandomization(self.mapname, self.hmodd.fullname, self.hmeven.fullname, self.workdir)
 
 
+    def cl_prediction(self):
+        """
+            To predicate contour level for both primary map and raw map output as json file
+        """
+
+        pred_primary = {}
+        pred_raw = {}
+        if self.map.fullname:
+            m = mrcfile.open(self.map.fullname)
+            d = m.data
+            norm_pred = calc_level_dev(d)[0]
+            pred_primary['primary'] = keep_three_significant_digits(float(norm_pred))
+
+        if self.rawmap:
+            m = mrcfile.open(self.rawmap.fullname)
+            d = m.data
+            norm_pred = calc_level_dev(d)[0]
+            pred_raw['raw'] = keep_three_significant_digits(float(norm_pred))
+
+        result_dict = {}
+        if pred_primary or pred_raw:
+            result_dict['predicated_contour_level'] = {**pred_primary, **pred_raw} if pred_primary and pred_raw \
+                                                                                    else pred_primary or pred_raw
+            print(result_dict)
+        else:
+            print('Predicated contour level does not exist for either primary or raw map.')
+
+        if result_dict:
+            file_name = f'{self.workdir}{self.mapname}_predicated_contour_level.json'
+            out_json(result_dict, file_name)
+
 
     def emringer(self):
         """

{emdbva-0.0.1.dev1 → emdbva-0.0.1.dev2}/va/version.py

@@ -28,4 +28,4 @@ under the License.
 
 """
 
-__version__ = '0.0.1.dev01'
+__version__ = '0.0.1.dev02'