emdbva 0.0.1.dev140__tar.gz → 0.0.1.dev144__tar.gz

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/MANIFEST.in

@@ -1,5 +1,6 @@
 include va/qscores.csv
 include va/utils/cl_weights.pth
+recursive-include va/schemas *.json
 exclude va/symmetry.py
 exclude va/onedepva.py
 exclude va/PATHS.py

{emdbva-0.0.1.dev140/emdbva.egg-info → emdbva-0.0.1.dev144}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev140
+Version: 0.0.1.dev144
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144/emdbva.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev140
+Version: 0.0.1.dev144
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/emdbva.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ emdbva.egg-info/dependency_links.txt
 emdbva.egg-info/entry_points.txt
 emdbva.egg-info/requires.txt
 emdbva.egg-info/top_level.txt
+va/PATHS_codon_airflow.py
 va/__init__.py
 va/mainva.py
 va/preparation.py
@@ -35,6 +36,9 @@ va/metrics/smoc.py
 va/metrics/strudel.py
 va/metrics/surfaces.py
 va/metrics/threedfsc.py
+va/schemas/__init__.py
+va/schemas/emdb_entry_full.schema.json
+va/schemas/loader.py
 va/utils/Checker.py
 va/utils/ChimeraxViews.py
 va/utils/MapProcessor.py

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/setup.py

@@ -19,8 +19,11 @@ setup(
     name="emdbva",
     version=__version__,
     packages=find_packages(),
+    package_data={
+        'va.schemas': ['*.json'],
+    },
     exclude_package_data={
-        'va': ['test/*', 'va/onedepva.py', 'core/*', 'proc/*', 'validation/*', 'test_data/*'],
+        'va': ['test/*', 'core/*', 'proc/*', 'validation/*', 'test_data/*'],
     },
     include_package_data=True,
     author="Zhe Wang",

emdbva-0.0.1.dev144/va/PATHS_codon_airflow.py

@@ -0,0 +1,80 @@
+"""
+PATHS.py
+
+
+PATHS is used to input and output directories of validation
+analysis.
+
+Copyright [2013] EMBL - European Bioinformatics Institute
+Licensed under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in
+compliance with the License. You may obtain a copy of
+the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+
+"""
+
+__author__ = 'Zhe Wang'
+__email__ = 'zhe@ebi.ac.uk'
+__date__ = '2022-06-14'
+
+# ENT file source path
+ENTSOURCE = '/nfs/msd/work2/ftp/pdb/data/structures/all/pdb/'
+
+# VTP file source path
+VTPSOURCE = '/nfs/pdbe_staging/pdbe_data/development/staging/em/'
+
+# CIF file source path
+# CIFSOURCE = '/nfs/production/gerard/pdbe/prerelease/current/ftp/pdb/data/structures/divided/mmCIF/'
+# CIFSOURCE = '/nfs/ftp/public/.staging/pdbe-G79Bblrm/projects/pdb/staging/data/structures/all/mmCIF/'
+# CIFSOURCE = '/hps/nobackup/pdbe/nfs_prod/ftp/data/structures/all/mmCIF/'
+CIFSOURCE = '/nfs/production/gerard/pdbe/ftp/pdb/data/structures/all/mmCIF/'
+
+MAP_SERVER_PATH = '/hps/nobackup/gerard/emdb/va/entry_results/'
+
+# Chimera path
+# CHIMERA = '/hps/nobackup/gerard/emdb/va/external/chimerax/usr/libexec/UCSF-ChimeraX/bin/ChimeraX'
+CHIMERA = '/hps/nobackup/gerard/emdb/va/external/chimera110/usr/libexec/UCSF-ChimeraX/bin/ChimeraX'
+
+# Original or old Chimera
+OCHIMERA = '/hps/nobackup/gerard/emdb/va/external/chimera/bin/chimera'
+
+# For ebi server to copy data for VA
+# VASOURCE = '/nfs/msd/em/ftp_rsync/staging/structures/'
+# VASOURCE = '/nfs/production/gerard/emdb/archive/staging/structures/'
+# VASOURCE = '/hps/nobackup/gerard/emdb/archive/local/structures/'
+VASOURCE = '/nfs/production/gerard/emdb/archive2/local/structures/'
+
+# VAPATH
+VAPATH = '/hps/nobackup/gerard/emdb/va_airflow/Validation-Analysis/va/'
+
+# Jsons and images are copied to here for display purpose
+FORDISPLAY = '/nfs/nobackup/msd/em_va/development/'
+
+# Proshade path
+PROSHADEPATH = '/hps/software/users/gerard/emdb/va/external/ccpem/ccpem-1.6.0/bin/proshade'
+
+# Meshmaker path
+MESHMAKERPATH = '/nfs/msd/em/software/meshmakertest/meshmaker/build/meshmaker'
+
+# VA production path
+VA_PROD_PATH = '/nfs/public/rw/pdbe/httpd-em/sessions-prod/validation_analysis'
+
+# VA staging path
+VA_STAG_PATH = '/nfs/public/rw/pdbe/httpd-em/sessions/validation_analysis'
+
+# Strudel lib
+LIB_STRUDEL_ROOT = '/hps/nobackup/gerard/emdb/va/external/strudel_motiflib/strudel-lib.4_0.5px'
+
+# 3DFSC fake to take positions
+THREEDFSC_ROOT = '/Users/zhe/Downloads/Anisotropy/ThreeDFSC/ThreeDFSC_Start.py'
+
+# ResMap
+# RESMAP = '/hps/nobackup/gerard/emdb/va/external/resmap/ResMap_project/ResMap.py'
+RESMAP = '/hps/software/users/gerard/emdb/va/external/ccpem/ccpem-1.6.0/bin/relion_postprocess'

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/va/mainva.py

@@ -3,8 +3,10 @@
 """
 mainva.py
 
-mainva class used to run the main function of validation
-analysis
+Entry points and orchestration helpers for the main VA pipeline.
+
+This module wires together preparation, validation analysis, and
+logging for the standard package-level VA workflow.
 
 Copyright [2013] EMBL - European Bioinformatics Institute
 Licensed under the Apache License, Version 2.0 (the
@@ -170,6 +172,7 @@ def inallruns(validationobj, runs):
     """
 
     def without_strudel():
+        """Run the selected validation steps other than Strudel."""
         # Projections
         if 'projection' in runs:
             validationobj.orthogonal_projections()
@@ -255,6 +258,7 @@ def inallruns(validationobj, runs):
         return None
 
     def strudel_only():
+        """Run the selected Strudel validation step."""
         # Strudel
         if 'strudel' in runs:
             validationobj.strudel()
@@ -333,6 +337,11 @@ def reconfigure_logging(temp_logger, buffer, log_file):
 
 # @profile
 def main():
+    """Run the main VA command-line entry point.
+
+    This function parses the CLI arguments, builds the preparation and
+    validation objects, and drives the configured validation runs.
+    """
 
     start_first = timeit.default_timer()
 
@@ -443,4 +452,3 @@ def main():
 
 if __name__ == '__main__':
     main()
-

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/va/metrics/bars.py

@@ -1,3 +1,5 @@
+"""Calculate score percentiles and generate metric comparison plots."""
+
 import os.path
 import json
 import pandas as pd
@@ -317,6 +319,22 @@ def plot_bar_mat(a, b, qmin, qmax, qscore, work_dir, plot_name, score_type, resb
 
 
 def bar(new_entry_dict, score_type, work_dir, score_dir, plot_name, update_bin_file=None):
+    """Calculate whole-dataset and resolution-relative score percentiles.
+
+    Args:
+        new_entry_dict: Score entry containing the selected score and resolution.
+        score_type: Score column to process.
+        work_dir: Directory for generated plot output.
+        score_dir: Directory containing the default score CSV.
+        plot_name: Plot filename used when a bar plot is generated.
+        update_bin_file: Optional CSV used to update the resolution-bin size.
+
+    Returns:
+        tuple: Whole-dataset statistics, relative-resolution statistics, and bin size.
+
+    Raises:
+        ValueError: If no usable score CSV is available.
+    """
     if update_bin_file and os.path.isfile(update_bin_file):
         input_file = update_bin_file
     else:
@@ -391,6 +409,11 @@ def score_whole(new_entry, df, score_type='qscore'):
 
 
 def score_relative(new_entry, df, score_type='qscore', column='resolution', bin=0.5):
+    """Calculate a score percentile within a resolution range.
+
+    Returns:
+        tuple: Relative percentile and number of rows in the selected range.
+    """
     df_relative = get_resolution_range(new_entry, df, score_type, column, bin)
     if df_relative.empty:  # Handle case where df_relative is empty
         return 0., 0.  # Return NaN for q_relative and an empty list for row_indices
@@ -401,10 +424,16 @@ def score_relative(new_entry, df, score_type='qscore', column='resolution', bin=
     return q_relative, rows
 
 def score_relative_wrapper(args):
+    """Unpack arguments for parallel relative-score calculation."""
     x, df, score_type, column, bin = args
     return score_relative(x, df, score_type, column, bin)
 
 def fast_qscore_resolution_bin(df_resolution_sorted, score_type='qscore'):
+    """Add relative-score columns for a series of resolution-bin sizes.
+
+    Returns:
+        pandas.DataFrame: The input DataFrame with calculated relative-score columns.
+    """
     values = [round(x, 1) for x in np.arange(0.1, 1.6, 0.1)]
     for value in values:
         args_list = [
@@ -417,6 +446,11 @@ def fast_qscore_resolution_bin(df_resolution_sorted, score_type='qscore'):
     return df_resolution_sorted
 
 def find_optimal_correlation_index(arr):
+    """Return the index of the value closest to -0.3 among eligible values.
+
+    Returns:
+        int or None: Selected index, or ``None`` if no value exceeds -0.3.
+    """
     arr = np.asarray(arr)
     mask = arr > -0.3
     if not np.any(mask):
@@ -426,6 +460,11 @@ def find_optimal_correlation_index(arr):
     return np.where(arr == closest_value)[0][0]
 
 def resolution_qrelative_correlation(df_resolution_sorted, values, score_type='qscore'):
+    """Calculate correlations between resolution and relative-score columns.
+
+    Returns:
+        tuple: Correlations below 5, correlations above 5, and column names.
+    """
     correlation_below_5 = []
     correlation_above_5 = []
     col_names = []
@@ -463,6 +502,7 @@ def resolution_qrelative_correlation(df_resolution_sorted, values, score_type='q
     return correlation_below_5, correlation_above_5, col_names
 
 def get_resolution_bin_size_fromva(score_file):
+    """Read a score CSV and return its recorded resolution-bin mode."""
     df = pd.read_csv(score_file, dtype={'id': str, 'resolution': float, 'name': str, 'qscore': float}, usecols=[0, 1, 2, 3])
     df = df.dropna()
     df_resolution_sorted = df.sort_values(by='resolution')

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/va/metrics/connected_percentage.py

@@ -1,12 +1,34 @@
+"""Measure connected and removed regions in thresholded MRC volumes."""
+
 import numpy as np
 import mrcfile
 from scipy.ndimage import label, find_objects
 
 
 def estimate_diameter_bbox(coords_physical):
+    """Estimate a point set's diameter from its axis-aligned bounding box.
+
+    Args:
+        coords_physical: Array of physical-space coordinates.
+
+    Returns:
+        float: Euclidean distance between opposite bounding-box corners.
+    """
     return np.linalg.norm(coords_physical.max(axis=0) - coords_physical.min(axis=0))
 
 def detect_small_blobs(volume, threshold, min_diameter, voxel_size=1.0, connectivity=2):
+    """Keep thresholded connected regions whose bounding-box diameter is large enough.
+
+    Args:
+        volume: Three-dimensional volume array.
+        threshold: Value used to create the binary foreground mask.
+        min_diameter: Minimum physical diameter to retain.
+        voxel_size: Physical size of one voxel.
+        connectivity: Connectivity mode used for labeling.
+
+    Returns:
+        tuple: Cleaned boolean mask and the number of removed voxels.
+    """
     binary = volume > threshold
     structure = np.ones((3, 3, 3)) if connectivity == 3 else None
     labeled, _ = label(binary, structure=structure)
@@ -26,6 +48,14 @@ def detect_small_blobs(volume, threshold, min_diameter, voxel_size=1.0, connecti
     return cleaned, total_removed_voxels
 
 def compute_connected_metrics(mrc_path, threshold, min_diameter=5.0, voxel_size=None, connectivity=2):
+    """Calculate connected and disconnected volume percentages from an MRC file.
+
+    The MRC volume is read from ``mrc_path``; the returned metrics are calculated
+    after removing regions below the requested diameter.
+
+    Returns:
+        dict: Connected and disconnected percentages and volumes.
+    """
     with mrcfile.open(mrc_path, permissive=True) as mrc:
         volume = mrc.data.copy()
         if voxel_size is None:

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/va/metrics/contour_level_predicator.py

@@ -1,3 +1,5 @@
+"""Predict contour levels from normalized three-dimensional map volumes."""
+
 import os
 os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
 import torch
@@ -14,7 +16,10 @@ warnings.filterwarnings("ignore", message="User provided device_type of 'cuda',
 
 # Convolutional layers adapted from https://www.nature.com/articles/s41598-022-19212-6
 class CNNModel(nn.Module):
+    """Three-dimensional convolutional model used for contour prediction."""
+
     def __init__(self):
+        """Initialize the convolutional and fully connected layers."""
         super(CNNModel, self).__init__()
         self.conv = nn.Sequential(
             nn.Conv3d(1, 32, 7, 1, padding_mode='replicate'),
@@ -42,6 +47,7 @@ class CNNModel(nn.Module):
                                  nn.Linear(in_features=64, out_features=1))
 
     def forward(self, x):
+        """Run a forward pass on a batch of 64-by-64-by-64 volumes."""
         x = x.view(-1, 1, 64, 64, 64)
         x = self.conv(x)
         x = x.view(x.size(0), -1)
@@ -50,10 +56,19 @@ class CNNModel(nn.Module):
 
 
 def check_boxsize(vol):
+    """Return the first dimension of a volume array."""
     return vol.shape[0]
 
 
 def load_data(vol_d_norm):
+    """Wrap a normalized volume in a single-item PyTorch data loader.
+
+    Args:
+        vol_d_norm: Normalized three-dimensional volume.
+
+    Returns:
+        DataLoader: Single-item loader for model prediction.
+    """
     vol_array = np.empty(shape=(1, 64, 64, 64), dtype=np.float32)
     vol_array[:] = vol_d_norm
     data_generator = torch.utils.data.DataLoader(vol_array, batch_size=1, shuffle=False)
@@ -61,6 +76,7 @@ def load_data(vol_d_norm):
 
 
 def downsample(vol):
+    """Downsample a volume to 64 cubed voxels in Fourier space."""
     boxsize = check_boxsize(vol)
     vol_ft = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(vol)))
     start = boxsize // 2 - 64 // 2
@@ -75,6 +91,7 @@ def downsample(vol):
 
 def upsample(
         vol):  # through interpolation of maps in Fourier space; based on https://www.sciencedirect.com/science/article/pii/S001046551400335X
+    """Pad a volume in Fourier space to the model's 64 cubed input size."""
     boxsize = check_boxsize(vol)
     new_box_size = 64
     vol_ft = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(vol)))
@@ -87,6 +104,11 @@ def upsample(
 
 
 def normalize(vol_data):
+    """Clip and normalize positive volume values for model input.
+
+    Returns:
+        tuple: Normalized volume and the lower and upper scaling values.
+    """
     d_upper = np.percentile(vol_data, 99.999)
     d_lower = np.percentile(vol_data, 0.001)
     vol_data = np.where(vol_data > d_upper, d_upper, vol_data)
@@ -109,6 +131,14 @@ model.load_state_dict(checkpoint['model_state_dict'])
 
 
 def model_pred(data_generator):
+    """Run the loaded model over a data loader without gradient tracking.
+
+    Args:
+        data_generator: Data loader containing normalized volume batches.
+
+    Returns:
+        list: Model prediction tensors collected for each batch.
+    """
     model.eval()
     pred_list = []
     with torch.no_grad():
@@ -122,6 +152,14 @@ def model_pred(data_generator):
 
 
 def calc_level_dev(vol_data):
+    """Predict contour levels after resizing the volume to model dimensions.
+
+    Args:
+        vol_data: Three-dimensional volume array.
+
+    Returns:
+        numpy.ndarray: Predicted contour-level values rescaled to input units.
+    """
     boxsize = check_boxsize(vol_data)
 
     if boxsize > 64:
@@ -154,4 +192,3 @@ def calc_level_dev(vol_data):
 
 
 
-

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/va/metrics/phaserandomization.py

@@ -1,3 +1,5 @@
+"""Create RELION masks and FSC inputs from map volumes."""
+
 from va.utils.ChimeraxViews import *
 from va.utils.misc import find_rawmap_file
 import numpy as np
@@ -57,6 +59,14 @@ def check_mrc(input_map):
 
 @profile_peak_memory()
 def relion_mask(raw_map, out_dir, mapname=None):
+    """Create a RELION mask and a locally generated comparison mask.
+
+    The function invokes the RELION mask executable when available and writes
+    the resulting mask into ``out_dir``.
+
+    Returns:
+        str or None: Generated mask path when successful.
+    """
     relion_mask_executable_name = 'relion_mask_create'
     relion_mask_executable = find_executable(relion_mask_executable_name)
     if relion_mask_executable:
@@ -140,7 +150,9 @@ def relion_fsc(mapone, maptwo, mask_file=None, out_dir=None):
 def relion_fsc_calculation(mapone, maptwo, root, mapname=None):
     """
         Calculates FSc using Relion
-    return: data dictionary of fsc
+
+    Returns:
+        dict: Data dictionary containing the FSC result.
     """
 
     result_type = 'fsc'
@@ -166,6 +178,7 @@ def relion_fsc_calculation(mapone, maptwo, root, mapname=None):
         return None
 
 def get_voxel_size(input_map):
+    """Return the x-axis voxel size read from an MRC map."""
 
     try:
         with mrcfile.open(input_map, permissive=True) as mrc:
@@ -193,6 +206,18 @@ def calculate_pixels(angpix):
 
 # following is the Python version of Relion auto masking
 def relion_auto_mask_fast(data, threshold, extend_pixels, edge_width, verbose=True):
+    """Build a soft-edged binary mask from an in-memory volume array.
+
+    Args:
+        data: Volume array to threshold.
+        threshold: Initial density threshold.
+        extend_pixels: Number of pixels to extend or shrink the mask.
+        edge_width: Width of the soft edge in pixels.
+        verbose: Whether to display progress messages.
+
+    Returns:
+        numpy.ndarray: Floating-point mask array.
+    """
     steps = []
     if verbose:
         steps = tqdm(total=3, desc="Masking Steps", unit="step")
@@ -239,6 +264,10 @@ def relion_auto_mask_fast(data, threshold, extend_pixels, edge_width, verbose=Tr
 
 @profile_peak_memory()
 def relion_auto_mask(input_file, output_file, threshold, extend_pixels, edge_width, verbose=True):
+    """Create an MRC mask file from an input MRC volume.
+
+    The input voxel size is copied to the output file.
+    """
     with mrcfile.open(input_file) as mrc:
         data = mrc.data.copy()
         voxel_size = mrc.voxel_size.copy()
@@ -251,4 +280,3 @@ def relion_auto_mask(input_file, output_file, threshold, extend_pixels, edge_wid
 
     if verbose:
         print(f"== Mask saved to {output_file} ==")
-

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/va/metrics/phenix_cc.py

@@ -146,8 +146,13 @@ def run_phenixcc(full_modelpath, full_mappath, resolution, output_path, echo=Tru
 def read_cc(cc_per_residue_log, errlist):
     """
         Read the residue-wise CCC from the output of Phenix results
-    :param: cc_per_residue_log input file cc_per_residue.log file(full path name) from Phenix CC calculation
-    :return:
+
+    Args:
+        cc_per_residue_log: Full path to the Phenix residue-wise CCC log.
+        errlist: Errors collected during the Phenix calculation.
+
+    Returns:
+        pandas.DataFrame or None: Parsed residue-wise CCC data when available.
     """
 
     if not errlist and os.path.isfile(cc_per_residue_log):
@@ -176,8 +181,12 @@ def _floatohex(numlist):
 def ccdf_todict(ccdf):
     """
         Given the Phenix CC result in dataframe and output the dict to be save in json
-    :param ccdf: phenix cc_per_residue.log result into dataframe
-    :return: dict which contain CC per residue results
+
+    Args:
+        ccdf: Phenix residue-wise CCC results in a dataframe.
+
+    Returns:
+        dict: Residue-wise and chain-level CC results.
     """
 
     finaldict = {}
@@ -208,13 +217,11 @@ def _safe_float(x):
 
 def parse_phenix_map_model_cc_output(stdout_text):
     """
-    Parse phenix.map_model_cc stdout and extract:
-      - overall CCs: CC_mask, CC_volume, CC_peaks, CC_box
-      - per-chain local CC table
-      - (optional) main-chain / side-chain summaries
-      - (optional) map resolution
-      - paths to logs that Phenix says it saved
-    Returns a dict; missing fields will be None or empty lists.
+    Parse phenix.map_model_cc stdout and extract overall CCs, per-chain local
+    CC values, optional chain summaries, map resolution, and reported log paths.
+
+    Returns:
+        dict: Parsed values; missing fields are ``None`` or empty lists.
     """
     result = {
         "resolution": None,

{emdbva-0.0.1.dev140 → emdbva-0.0.1.dev144}/va/metrics/phenix_mm.py

@@ -55,8 +55,13 @@ def run_phenixmmfsc(full_modelpath, full_mappath, output_path):
 def read_mmfsc(mmfsc_log, errlist):
     """
         Read the residue-wise CCC from the output of Phenix results
-    :param: cc_per_residue_log input file cc_per_residue.log file(full path name) from Phenix CC calculation
-    :return:
+
+    Args:
+        mmfsc_log: Full path to the Phenix map-model FSC log.
+        errlist: Errors collected during the Phenix calculation.
+
+    Returns:
+        pandas.DataFrame or None: Parsed data when available.
     """
 
     if not errlist and os.path.isfile(mmfsc_log):
@@ -145,8 +150,13 @@ def all_intersection(allcurves):
 def mmfscdf_todict(mmfscdf, nyquist):
     """
         Given the Phenix CC result in dataframe and output the dict to be save in json
-    :param ccdf: phenix cc_per_residue.log result into dataframe
-    :return: dict which contain CC per residue results
+
+    Args:
+        mmfscdf: Phenix map-model FSC results in a dataframe.
+        nyquist: Maximum frequency value to retain.
+
+    Returns:
+        dict: FSC curves and their intersections.
     """
 
     real_mmfscdf = mmfscdf[mmfscdf[0] <= nyquist].round(3).drop_duplicates()
@@ -168,4 +178,3 @@ def mmfscdf_todict(mmfscdf, nyquist):
     finaldict = {'curves': curves, 'intersections': intersections}
 
     return finaldict
-