scipion-em-xmipp 23.7.0.0__tar.gz → 23.11.0.1__tar.gz

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/CHANGES.md

@@ -1,5 +1,30 @@
+## Release 3.23.11 - Nereus
+   - New protocols
+      - Volume local adjustment
+   - Protocols updated
+      - convert_pdb: Allowed conversion natively from CIFs
+      - particle_pick_automatic: The model can now be given by a directory
+      - volume_local_adjust: Save occupancy volume
+      - extract_particles: Added two different cases for downsampling, by dimensions and by sampling rate
+   - Protocols fixed
+      - movie_resize: Fixed movie resize output size
+      - movie_gain: Fix update output step by using a generic one from scipion
+      - tilt_analysis: Fixes in the generated tilt images and in updating correctly the output sets
+      - ctf_consensus: Fix the dependencies of the step
+      - preprocess_micrographs: Fixed output size in preprocess micrographs
+      - deep_center_assignment: Fixed deep center calls
+      - extract_particles_movies: Get coords correctly
+      - particle_pick_consensus: Fix MicsPointer
+      - trigger_data: fix updateOutput and close correclty the output set
+   - Protocols deprecated (For more details visit [this](https://github.com/I2PC/xmipp/wiki/Deprecating-programs-and-protocols))
+      - classification_gpuCorr
+      - classification_gpuCorr_full
+      - classification_gpuCorr_semi
+   - More scipio-em-xmipp
+      - Updated Nvidia driver required version
+
 
-## Release 3.23.07 - 
+## Release 3.23.07 - Morpheus 
    - New protocols
       - Movie Dose analysis
       - deep_center

{scipion-em-xmipp-23.7.0.0/scipion_em_xmipp.egg-info → scipion-em-xmipp-23.11.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: scipion-em-xmipp
-Version: 23.7.0.0
+Version: 23.11.0.1
 Summary: Plugin to use Xmipp programs within the Scipion framework (see https://scipion-em.github.io/docs/).
 Home-page: https://github.com/i2pc/scipion-em-xmipp
 Author: I2PC

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1/scipion_em_xmipp.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: scipion-em-xmipp
-Version: 23.7.0.0
+Version: 23.11.0.1
 Summary: Plugin to use Xmipp programs within the Scipion framework (see https://scipion-em.github.io/docs/).
 Home-page: https://github.com/i2pc/scipion-em-xmipp
 Author: I2PC

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/scipion_em_xmipp.egg-info/SOURCES.txt

@@ -38,9 +38,6 @@ xmipp3/protocols/protocol_center_particles.py
 xmipp3/protocols/protocol_cl2d.py
 xmipp3/protocols/protocol_cl2d_align.py
 xmipp3/protocols/protocol_classes_2d_mapping.py
-xmipp3/protocols/protocol_classification_gpuCorr.py
-xmipp3/protocols/protocol_classification_gpuCorr_full.py
-xmipp3/protocols/protocol_classification_gpuCorr_semi.py
 xmipp3/protocols/protocol_compare_angles.py
 xmipp3/protocols/protocol_compare_reprojections.py
 xmipp3/protocols/protocol_consensus_classes.py
@@ -52,11 +49,8 @@ xmipp3/protocols/protocol_ctf_consensus.py
 xmipp3/protocols/protocol_ctf_correct_wiener2d.py
 xmipp3/protocols/protocol_ctf_defocus_group.py
 xmipp3/protocols/protocol_ctf_micrographs.py
-xmipp3/protocols/protocol_deep_center.py
-xmipp3/protocols/protocol_deep_center_predict.py
+xmipp3/protocols/protocol_deep_center_assignment.py
 xmipp3/protocols/protocol_deep_denoising.py
-xmipp3/protocols/protocol_deep_global_assignment.py
-xmipp3/protocols/protocol_deep_global_assignment_predict.py
 xmipp3/protocols/protocol_deep_hand.py
 xmipp3/protocols/protocol_deep_micrograph_screen.py
 xmipp3/protocols/protocol_denoise_particles.py
@@ -124,6 +118,7 @@ xmipp3/protocols/protocol_validate_overfitting.py
 xmipp3/protocols/protocol_volume_adjust_sub.py
 xmipp3/protocols/protocol_volume_consensus.py
 xmipp3/protocols/protocol_volume_deform_zernike3d.py
+xmipp3/protocols/protocol_volume_local_adjust.py
 xmipp3/protocols/protocol_volume_local_sharpening.py
 xmipp3/protocols/protocol_volume_strain.py
 xmipp3/protocols/protocol_write_testC.py
@@ -152,11 +147,8 @@ xmipp3/tests/test_protocol_apply_transformation_matrix.py
 xmipp3/tests/test_protocol_compare_angles.py
 xmipp3/tests/test_protocol_consensus_classes3D.py
 xmipp3/tests/test_protocol_ctf_consensus.py
-xmipp3/tests/test_protocol_deep_center.py
-xmipp3/tests/test_protocol_deep_center_predict.py
+xmipp3/tests/test_protocol_deep_center_assignment.py
 xmipp3/tests/test_protocol_deep_denoising.py
-xmipp3/tests/test_protocol_deep_global_assignment.py
-xmipp3/tests/test_protocol_deep_global_assignment_predict.py
 xmipp3/tests/test_protocol_extract_asymmetric_unit.py
 xmipp3/tests/test_protocol_monodir.py
 xmipp3/tests/test_protocol_multiple_fsc.py
@@ -172,9 +164,6 @@ xmipp3/tests/test_protocols_continuousflex.py
 xmipp3/tests/test_protocols_deepVolPostprocessing.py
 xmipp3/tests/test_protocols_deepres.py
 xmipp3/tests/test_protocols_fso.py
-xmipp3/tests/test_protocols_gpuCorr_classifier.py
-xmipp3/tests/test_protocols_gpuCorr_fullStreaming.py
-xmipp3/tests/test_protocols_gpuCorr_semiStreaming.py
 xmipp3/tests/test_protocols_highres.py
 xmipp3/tests/test_protocols_local_defocus.py
 xmipp3/tests/test_protocols_localdeblur.py

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/__init__.py

@@ -37,12 +37,15 @@ import pyworkflow.utils as pwutils
 from .base import *
 from .constants import XMIPP_HOME, XMIPP_URL, XMIPP_DLTK_NAME
 
-_logo = "xmipp_logo.png"
-_references = ['delaRosaTrevin2013', 'Sorzano2013', 'Strelak2021']
-_currentBinVersion = '3.23.07.0'
-__version__ = _currentBinVersion[2:] + ".0"  # Set this to ".0" on each xmipp binary release, otherwise increase it --> ".1", ".2", ...
+type_of_version = 'release' #'devel'
+_logo = "xmipp_logo" + ("" if type_of_version == 'release' else '_devel') + '.png'
 
+_references = ['delaRosaTrevin2013', 'Sorzano2013', 'Strelak2021']
+_currentBinVersion = '3.23.11.0'
+__version__ = _currentBinVersion[2:] + ".1"  # Set this to ".0" on each xmipp binary release, otherwise increase it --> ".1", ".2", ...
 
+# Requirement version variables
+NVIDIA_DRIVERS_MINIMUM_VERSION = 450
 
 class Plugin(pwem.Plugin):
     _homeVar = XMIPP_HOME
@@ -201,6 +204,7 @@ def installDeepLearningToolkit(plugin, env):
     preMsgs = []
     cudaMsgs = []
     nvidiaDriverVer = None
+    useGpu = False
     if os.environ.get('CUDA', 'True') == 'True':
         try:
             nvidiaDriverVer = subprocess.Popen(["nvidia-smi",
@@ -209,12 +213,12 @@ def installDeepLearningToolkit(plugin, env):
                                                env=plugin.getEnviron(),
                                                stdout=subprocess.PIPE
                                                ).stdout.read().decode('utf-8').split(".")[0]
-            if int(nvidiaDriverVer) < 390:
+            if int(nvidiaDriverVer) < NVIDIA_DRIVERS_MINIMUM_VERSION:
                 preMsgs.append("Incompatible driver %s" % nvidiaDriverVer)
-                cudaMsgs.append("Your NVIDIA drivers are too old (<390). "
+                cudaMsgs.append(f"Your NVIDIA drivers are too old (<{NVIDIA_DRIVERS_MINIMUM_VERSION}). "
                                 "Tensorflow was installed without GPU support. "
                                 "Just CPU computations enabled (slow computations)."
-                                "To enable CUDA (drivers>390 needed), "
+                                f"To enable CUDA (drivers>{NVIDIA_DRIVERS_MINIMUM_VERSION} needed), "
                                 "set CUDA=True in 'scipion.conf' file")
                 nvidiaDriverVer = None
         except (ValueError, TypeError):
@@ -239,7 +243,7 @@ def installDeepLearningToolkit(plugin, env):
     cmdsInstall = list(CondaEnvManager.yieldInstallAllCmds(useGpu=useGpu))
 
     now = datetime.now()
-    installDLvars = {'modelsUrl': "http://scipion.cnb.csic.es/downloads/scipion/software/em",
+    installDLvars = {'modelsUrl': "https://scipion.cnb.csic.es/downloads/scipion/software/em",
                      'syncBin': plugin.getHome('bin/xmipp_sync_data'),
                      'modelsDir': plugin.getHome('models'),
                      'modelsPrefix': "models_UPDATED_on",

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/bibtex.py

@@ -620,5 +620,43 @@ DOI = {https://doi.org/10.3390/molecules26206224}
 }
 
 
+@article{Fernandez-Gimenez2023a,
+title = {A new algorithm for particle weighted subtraction to eliminate signals from unwanted components in Single Particle Analysis},
+journal = {Journal of Structural Biology},
+volume = {215},
+number = {4},
+year = {2023},
+issn = {1047-8477},
+doi = {https://doi.org/10.1016/j.jsb.2023.108024},
+url = {https://www.sciencedirect.com/science/article/pii/S1047847723000874},
+author = {E. Fernández-Giménez, M. Martínez, R. Marabini, D. Strelak, R. Sánchez-García, J.M. Carazo and C.O.S. Sorzano},
+}
+
+@article{Fernandez-Gimenez2023b,
+title = {Local defocus estimation in Single Particle Analysis in Cryo-Electron Microscopy},
+journal = {Journal of Structural Biology},
+volume = {215},
+number = {4},
+year = {2023},
+issn = {1047-8477},
+doi = {https://doi.org/10.1016/j.jsb.2023.108030},
+url = {https://www.sciencedirect.com/science/article/pii/S104784772300093X},
+author = {E. Fernández-Giménez, J.M. Carazo and C.O.S. Sorzano},
+
+
+@Article{Strelak2023performance,
+AUTHOR = {Střelák, David and Marchán, Daniel and Carazo, José María and S. Sorzano, Carlos O.},
+TITLE = {Performance and Quality Comparison of Movie Alignment Software for Cryogenic Electron Microscopy},
+JOURNAL = {Micromachines},
+VOLUME = {14},
+YEAR = {2023},
+NUMBER = {10},
+ARTICLE-NUMBER = {1835},
+URL = {https://www.mdpi.com/2072-666X/14/10/1835},
+ISSN = {2072-666X},
+DOI = {https://doi.org/10.3390/mi14101835}
+
+}
+
 """
 

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/convert/io_coordinates.py

@@ -17,9 +17,10 @@ def readSetOfCoordsFromPosFnames(posDir, setOfInputCoords, sqliteOutName,
     """
 
     inputMics = setOfInputCoords.getMicrographs()
+    inputMicsPointer = setOfInputCoords.getMicrographs(asPointer=True)
     cleanPath(sqliteOutName)
     setOfOutputCoordinates= SetOfCoordinates(filename= sqliteOutName)
-    setOfOutputCoordinates.setMicrographs(inputMics)
+    setOfOutputCoordinates.setMicrographs(inputMicsPointer)
     setOfOutputCoordinates.setBoxSize(setOfInputCoords.getBoxSize())
     readSetOfCoordinates(posDir, micSet=inputMics,
                            coordSet=setOfOutputCoordinates,

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/protocols/__init__.py

@@ -40,9 +40,6 @@ from .protocol_break_symmetry import XmippProtAngBreakSymmetry
 from .protocol_cl2d_align import XmippProtCL2DAlign
 from .protocol_cl2d import XmippProtCL2D
 #from .protocol_classify_kmeans2d import XmippProtKmeansClassif2D
-from .protocol_classification_gpuCorr import XmippProtGpuCrrCL2D
-from .protocol_classification_gpuCorr_semi import XmippProtStrGpuCrrSimple
-from .protocol_classification_gpuCorr_full import XmippProtStrGpuCrrCL2D
 from .protocol_ctf_defocus_group import XmippProtCTFDefocusGroup
 from .protocol_compare_reprojections import XmippProtCompareReprojections
 from .protocol_compare_angles import XmippProtCompareAngles
@@ -136,9 +133,7 @@ from .protocol_structure_map import XmippProtStructureMap
 from .protocol_apply_zernike3d import XmippApplyZernike3D
 from .protocol_volume_adjust_sub import XmippProtVolAdjust, XmippProtVolSubtraction
 from .protocol_volume_consensus import XmippProtVolConsensus
+from .protocol_volume_local_adjust import XmippProtLocalVolAdj
 from .protocol_classes_2d_mapping import XmippProtCL2DMap
 from .protocol_deep_hand import XmippProtDeepHand
-from .protocol_deep_center import XmippProtDeepCenter
-from .protocol_deep_center_predict import XmippProtDeepCenterPredict
-from .protocol_deep_global_assignment import XmippProtDeepGlobalAssignment
-from .protocol_deep_global_assignment_predict import XmippProtDeepGlobalAssignmentPredict
+from .protocol_deep_center_assignment import XmippProtDeepCenter, XmippProtDeepGlobalAssignment

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/protocols/protocol_align_volume.py

@@ -52,7 +52,7 @@ class XmippProtAlignVolume(ProtAlignVolume):
      """
     _label = 'align volume'
     nVols = 0
-    _devStatus = UPDATED
+    _devStatus = PROD
 
     
     def __init__(self, **args):

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/protocols/protocol_analyze_local_ctf.py

@@ -34,6 +34,8 @@ from pwem.protocols import ProtAnalysis3D
 import pwem.emlib.metadata as md
 from xmipp3.convert import readSetOfMicrographs, writeSetOfMicrographs
 
+CITE ='Fernandez-Gimenez2023b'
+
 
 class XmippProtAnalyzeLocalCTF(ProtAnalysis3D):
     """Assigns to each micrograph a coefficient (R2) which evaluates the result of the

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/protocols/protocol_consensus_classes.py

@@ -164,18 +164,6 @@ class XmippProtConsensusClasses(ProtClassify3D):
         self._writeElbows(elbows)
 
     # --------------------------- INFO functions -------------------------------
-    """
-    def _validate(self):
-        errors = []
-
-        # Ensure that all classifications are made of the same images
-        items = self._getInputImages(0)
-        for i in range(1, len(self.inputClassifications)):
-            if self._getInputImages(i) != items:
-                errors.append(f'Classification {i} has been done with different images')
-
-        return errors
-    """
     
     # --------------------------- UTILS functions ------------------------------
     def _getInputClassificationCount(self) -> int:
@@ -479,7 +467,6 @@ class XmippProtConsensusClasses(ProtClassify3D):
             mu2 = np.mean(d2)
             sigma1 = np.std(d1, ddof=1)
             sigma2 = np.std(d2, ddof=1)
-            #sigma = (len(d1)*sigma1 + len(d2)*sigma2) / (len(d1) + len(d2)) # Weighted average
         
             # Compute the log likelihood
             logLikelihoods1 = scipy.stats.norm.logpdf(d1, mu1, sigma1)

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/protocols/protocol_consensus_local_ctf.py

@@ -40,11 +40,12 @@ from xmipp3.convert import setXmippAttribute
 from pyworkflow import BETA, UPDATED, NEW, PROD
 
 OUTPUTNAME = "outputParticles"
+CITE ='Fernandez-Gimenez2023b'
 
 class XmippProtConsensusLocalCTF(ProtAnalysis3D):
     """This protocol compares the estimations of local defocus computed by different protocols for a set of particles"""
     _label = 'consensus local defocus'
-    _devStatus = UPDATED
+    _devStatus = PROD
 
     _lastUpdateVersion = VERSION_2_0
     _possibleOutputs = {OUTPUTNAME:SetOfParticles}

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/protocols/protocol_convert_pdb.py

@@ -28,6 +28,9 @@
 # *
 # **************************************************************************
 
+# General imports
+import os
+
 # Scipion em imports
 from pwem.convert.headers import setMRCSamplingRate
 from pwem.emlib.image import ImageHandler
@@ -37,7 +40,7 @@ from pwem.protocols import ProtInitialVolume
 import pyworkflow.protocol.params as params
 import pyworkflow.protocol.constants as const
 from pyworkflow.utils import replaceBaseExt, removeExt, getExt, createLink, replaceExt, removeBaseExt
-from pyworkflow import BETA, UPDATED, NEW, PROD
+from pyworkflow import UPDATED
 
 class XmippProtConvertPdb(ProtInitialVolume):
     """ Convert atomic structure(s) into volume(s) """
@@ -70,7 +73,7 @@ class XmippProtConvertPdb(ProtInitialVolume):
         coordsCondition = 'setSize and not vol'
 
         # Defining parallel arguments
-        form.addParallelSection(threads=4)
+        form.addParallelSection(threads=4, mpi=1)
 
         # Generating form
         form.addSection(label='Input')
@@ -102,12 +105,16 @@ class XmippProtConvertPdb(ProtInitialVolume):
         form.addParam('centerPdb', params.BooleanParam, default=True,
                       expertLevel=const.LEVEL_ADVANCED, 
                       label="Center PDB",
-                      help='Center PDB with the center of mass')
+                      help='Center PDB with the center of mass.')
         form.addParam('outPdb', params.BooleanParam, default=False, 
                       expertLevel=const.LEVEL_ADVANCED, 
                       label="Store centered PDB",
                       help='Set to \'Yes\' if you want to save centered PDB. '
-                           'It will be stored in the output directory of this protocol')
+                           'It will be stored in the output directory of this protocol.')
+        form.addParam('convertCif', params.BooleanParam, default=True,
+                      expertLevel=const.LEVEL_ADVANCED, 
+                      label="Convert CIF to PDB",
+                      help='If set to true and input atom struct file is a CIF, it will get converted to PDB.')
         form.addParam('clean', params.BooleanParam, default=True,
                       expertLevel=const.LEVEL_ADVANCED, 
                       label="Clean tmp files",
@@ -138,8 +145,8 @@ class XmippProtConvertPdb(ProtInitialVolume):
     # --------------------------- STEPS functions --------------------------------------------
     def processConversion(self, pdb, samplingR, isSet):
         """ This step runs the pdb conversion. """
-        # Converting all input atomic structures to .pdb
-        pdb = self._convertToPdb(pdb)
+        # Conditionally converting all input atomic structures to .pdb
+        pdb = self._convertAtomStruct(pdb)
         
         # Generating output file for each input
         outFile = removeExt(pdb)
@@ -260,13 +267,14 @@ class XmippProtConvertPdb(ProtInitialVolume):
             # If it is a SetOfAtom Structs, get all of the elements iterating the set
             return [i.getFileName() for i in pbdObj]
 
-    def _convertToPdb(self, pdbRaw):
-        """ This function receives an atomic struct file, and converts it to .pdb if it is in .cif format. """
-        # Get output path for pdb file
-        convertedPdb = self._getExtraPath(replaceBaseExt(pdbRaw, 'pdb')).replace(" ", "_")
+    def _convertAtomStruct(self, pdbRaw):
+        """ This function receives an atomic struct file, and conditionally converts it to .pdb if it is in .cif format. """
+        # Get output path for atomic struct file
+        baseExt = replaceBaseExt(pdbRaw, 'pdb') if self.convertCif.get() else os.path.basename(pdbRaw)
+        convertedPdb = self._getExtraPath(baseExt).replace(" ", "_")
 
-        # If file extension is .cif, convert to .pdb, or else we link it as is
-        if getExt(pdbRaw) == ".cif":
+        # If file extension is .cif and conversion is required, convert to .pdb, or else we link it as is
+        if getExt(pdbRaw) == ".cif" and self.convertCif.get():
             cifToPdb(pdbRaw, convertedPdb)
         else:
             createLink(pdbRaw, convertedPdb)

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/protocols/protocol_create_gallery.py

@@ -51,7 +51,7 @@ class XmippProtCreateGallery(ProtAnalysis3D):
         form.addParam('symmetryGroup', StringParam, default="c1",
                       label='Symmetry group', 
                       help='See'
-                           'http://xmipp.cnb.uam.es/twiki/bin/view/Xmipp/Symmetry '
+                           'https://github.com/I2PC/xmipp-portal/wiki/Symmetry '
                            'for a description of the symmetry groups format. '
                            'If no symmetry is present, give c1')
 

{scipion-em-xmipp-23.7.0.0 → scipion-em-xmipp-23.11.0.1}/xmipp3/protocols/protocol_ctf_consensus.py

@@ -59,7 +59,7 @@ class XmippProtCTFConsensus(ProtCTFMicrographs):
     the agreement with a secondary CTF for the same set of micrographs.
     """
     _label = 'ctf consensus'
-    _devStatus = UPDATED
+    _devStatus = PROD
     _lastUpdateVersion = VERSION_3_0
 
     def __init__(self, **args):
@@ -200,12 +200,11 @@ class XmippProtCTFConsensus(ProtCTFMicrographs):
             self.ctfFn2 = self.inputCTF2.get().getFileName()
             self.allCtf2 = {}
 
-        ctfSteps = self._checkNewInput()
         self._insertFunctionStep('createOutputStep',
-                                 prerequisites=ctfSteps, wait=True)
+                                 prerequisites=[], wait=True)
 
     def createOutputStep(self):
-        pass
+        self._closeOutputSet()
 
     def initializeParams(self):
         self.finished = False