scipion-pyworkflow 3.7.0__py3-none-any.whl

pyworkflow/project/scripts/clean_projects.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *
+# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia, CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+import sys
+
+import pyworkflow as pw
+from pyworkflow.project import Manager
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+    
+    Usage: scipion python -m pyworkflow.project.scripts.clean_projects [SCIPION_USER_DATA] [--delete]
+        Clean projects that have expired (now - creation time > life time)
+        Optional to pass SCIPION_USER_DATA folder from which to read 'projects'.
+        If --delete is not passed, only a message with the projects to be deleted
+        will be shown. If used --delete, the projects will be deleted from filesystem.
+    """ % error)
+    sys.exit(1)
+
+
+n = len(sys.argv)
+
+if n > 3:
+    usage("Incorrect number of input parameters")
+    
+delete = '--delete' in sys.argv
+
+arg1 = sys.argv[1]
+if n > 1 and arg1 != '--delete':
+    customUserData = arg1
+else:
+    customUserData = pw.Config.SCIPION_USER_DATA
+
+print("Loading projects from:\n %s" % customUserData)
+ 
+# Create a new project
+manager = Manager(workspace=customUserData)
+
+for projInfo in manager.listProjects():
+    projName = projInfo.getName()
+    proj = manager.loadProject(projName)
+    settings = proj.getSettings()
+    
+    leftTime = proj.getLeftTime()
+    if (leftTime is not None and 
+            leftTime.days < 0):
+        if delete:
+            print("Deleting: %s (%s) " % (projName, leftTime))
+            manager.deleteProject(projName)
+        else: 
+            print("Should delete: %s (%s)" % (projName, leftTime))

pyworkflow/project/scripts/config.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *
+# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia, CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+
+import sys
+import os
+
+import pyworkflow as pw
+from pyworkflow.project import Manager
+from pyworkflow.object import Boolean
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+    
+    Usage: scipion python -m pyworkflow.project.scripts.config ProjectName [readOnly=True|False] [lifeTime=X|None]
+        This script shows (or edits) some of the configuration of the project.
+        Use readOnly=True (or False) to set/unset read-only property.
+        Use lifeTime=X for setting X hours or None to unset lifetime of the project.
+    """ % error)
+    sys.exit(1)    
+
+
+n = len(sys.argv)
+
+if n < 2 or n > 4:
+    usage("Incorrect number of input parameters")
+    
+# Load the given project
+projectsDir = os.path.join(pw.Config.SCIPION_USER_DATA, 'projects')
+projName = sys.argv[1]
+manager = Manager()
+project = manager.loadProject(projName)
+
+if project is None:
+    usage("Project '%s' does not exist in: \n  %s" % (projName, projectsDir))
+
+setReadOnly = False
+setLifeTime = False
+
+for arg in sys.argv:
+    if arg.startswith('readOnly='):
+        setReadOnly = True
+        value = arg.split('readOnly=')[1]
+        b = Boolean(value=value)
+        readOnlyValue = b.get()
+    elif arg.startswith('lifeTime='):
+        setLifeTime = True
+        value = arg.split('lifeTime=')[1]
+        lifeTimeValue = None if value == 'None' else int(value)
+
+if setReadOnly:
+    project.setReadOnly(readOnlyValue)
+    
+if setLifeTime:
+    project.settings.setLifeTime(lifeTimeValue)
+    
+if setReadOnly or setLifeTime:
+    # Truly write settings
+    project.settings.write()
+
+print("Projects: ", projectsDir)
+print("Project name: ", projName)
+print(" Settings: ")
+print("   readOnly = %s" % project.isReadOnly())
+print("   lifeTime = %s (hours)" % project.settings.getLifeTime())
+

pyworkflow/project/scripts/create.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# **************************************************************************
+# *
+# * Authors:     Pablo Conesa (pconesa@cnb.csic.es)
+# *
+# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia, CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+import sys
+import os
+
+from pyworkflow.project import Manager
+import pyworkflow.utils as pwutils
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+
+    Usage: scipion python -m pyworkflow.project.scripts.create NAME [WORKFLOW] [LOCATION]
+        NAME: project name
+        WORKFLOW: path to a Scipion json workflow
+        LOCATION: where to create it, defaults to scipion default location
+
+        This script will create a project project, optionally based on a workflow file
+    """ % error)
+    sys.exit(1)
+
+
+n = len(sys.argv)
+
+if n < 2 or n > 4:
+    usage("Incorrect number of input parameters")
+
+projName = sys.argv[1]
+
+jsonFile = None if n < 3 else os.path.abspath(sys.argv[2])
+location = None if n < 4 else sys.argv[3]
+
+# This might not be working anymore for python3.
+# I'm getting invalid ELF header triggered by matplotlib -->from . import _tkagg
+# path = pw.join('gui', 'no-tkinter')
+# sys.path.insert(1, path)
+
+# Create a new project
+manager = Manager()
+
+if manager.hasProject(projName):
+    usage("There is already a project with this name: %s"
+          % pwutils.red(projName))
+
+if jsonFile is not None and not os.path.exists(jsonFile):
+    usage("Nonexistent json file: %s" % pwutils.red(jsonFile))
+
+project = manager.createProject(projName, location=location)
+
+if jsonFile is not None:
+    protDict = project.loadProtocols(jsonFile)

pyworkflow/project/scripts/edit_workflow.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
+# *
+# * [1] SciLifeLab, Stockholm University
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+
+import sys
+import os
+from datetime import datetime
+
+import pyworkflow as pw
+import pyworkflow.utils as pwutils
+from pyworkflow.project import Manager
+from pyworkflow.gui.project import ProjectWindow
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+    
+    Usage: scipion python -m pyworkflow.project.scripts.edit_workflow workflow.json
+        Edit the provide json file with scipion workflow.
+        It will create a project, import the workflow and save
+        the workflow back before closing the project.
+        After that, the project will be deleted.
+    """ % error)
+    sys.exit(1)    
+
+
+n = len(sys.argv)
+
+if n < 2 or n > 3:
+    usage("Incorrect number of input parameters")
+    
+jsonFn = os.path.abspath(sys.argv[1])
+
+now = datetime.now()
+tempSpace = "editor-%s" % now.strftime('%Y%m%d-%H%M%S')
+customUserData = os.path.join(pw.Config.SCIPION_USER_DATA, 'tmp', tempSpace)
+
+pwutils.makePath(os.path.join(customUserData, 'projects'))
+
+print("Loading projects from:\n", customUserData)
+ 
+# Create a new project
+manager = Manager(workspace=customUserData)
+
+projName = os.path.basename(jsonFn)
+proj = manager.createProject(projName)
+projPath = manager.getProjectPath(proj.getShortName())
+proj.loadProtocols(jsonFn)
+
+
+class EditorProjectWindow(ProjectWindow):
+    def close(self, e=None):
+        try:
+            print("Writing protocols to: ", jsonFn)
+            proj.getRunsGraph(refresh=True)  # Build project runs graph
+            proj.exportProtocols(proj.getRuns(), jsonFn)
+            print("Deleting temporary folder: ", customUserData)
+            pwutils.cleanPath(customUserData)
+        except Exception as ex:
+            print("Error saving the workflow: ", ex)
+        ProjectWindow.close(self, e)
+
+
+projWindow = EditorProjectWindow(projPath)
+projWindow.show()

pyworkflow/project/scripts/fix_links.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import logging
+logging.basicConfig(level="INFO")
+
+from pyworkflow.project import Manager
+import pyworkflow.utils as pwutils
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+    
+    Usage: scipion python -m pyworkflow.project.scripts.fix_links PROJECT SEARCH_DIR
+        PROJECT: provide the project name to fix broken links in the imports
+        SEARCH_DIR: provide a directory where to look for the files and fix the links    
+    """ % error)
+    sys.exit(1)
+
+
+if len(sys.argv) != 3:
+    usage("Incorrect number of input parameters")
+
+projName = sys.argv[1]
+searchDir = os.path.abspath(sys.argv[2])
+
+# Create a new project
+manager = Manager()
+
+if not manager.hasProject(projName):
+    usage("Nonexistent project: %s" % pwutils.red(projName))
+
+if not os.path.exists(searchDir):
+    usage("Nonexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
+
+project = manager.loadProject(projName)
+project.fixLinks(searchDir)

pyworkflow/project/scripts/load.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
+# *
+# * [1] SciLifeLab, Stockholm University
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+
+import sys
+import os
+from datetime import datetime
+
+import pyworkflow as pw
+import pyworkflow.utils as pwutils
+from pyworkflow.project import Manager
+from pyworkflow.gui.project import ProjectWindow
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+    
+    Usage: scipion python -m pyworkflow.project.scripts.load /path/to/project
+        This script allows to quickly load a project folder without
+        importing that in the general user data folder
+    """ % error)
+    sys.exit(1)    
+
+
+n = len(sys.argv)
+
+if n < 2 or n > 3:
+    usage("Incorrect number of input parameters")
+    
+pathToProj = os.path.abspath(sys.argv[1])
+
+now = datetime.now()
+tempSpace = "loading-%s" % now.strftime('%Y%m%d-%H%M%S')
+customUserData = os.path.join(pw.Config.SCIPION_USER_DATA, 'tmp', tempSpace)
+
+projectsDir = os.path.join(customUserData, 'projects')
+pwutils.makePath(projectsDir)
+
+print("Loading projects from:\n", projectsDir)
+
+projName = os.path.basename(pathToProj)
+pwutils.createAbsLink(pathToProj, os.path.join(projectsDir, projName))
+ 
+# Create a new project
+manager = Manager(workspace=customUserData)
+
+proj = manager.loadProject(projName)
+projPath = manager.getProjectPath(projName)
+
+
+class EditorProjectWindow(ProjectWindow):
+    def close(self, e=None):
+        try:
+            print("Deleting temporary folder: ", customUserData)
+            pwutils.cleanPath(customUserData)
+        except Exception as ex:
+            print("Error: ", ex)
+        ProjectWindow.close(self, e)
+
+
+projWindow = EditorProjectWindow(projPath)
+projWindow.show()

pyworkflow/project/scripts/refresh.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# **************************************************************************
+# *
+# * Authors:     Yaiza Rancel (yrancel@cnb.csic.es)
+# *
+# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia, CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+import sys
+import os
+from time import sleep
+
+import pyworkflow as pw
+from pyworkflow.project import Manager
+from pyworkflow.project import Project
+import pyworkflow.utils as pwutils
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+
+    Usage: scipion python -m pyworkflow.scripts.project.refresh project_name [refresh_period]
+        This script will refresh the project.sqlite file of the specified project.
+        We can specify how often it will refresh in seconds (defaults 60).
+        e.g.
+        scipion python -m pyworkflow.scripts.project.refresh TestExtractParticles 15
+    """ % error)
+    sys.exit(1)
+
+
+n = len(sys.argv)
+
+if n > 3:
+    usage("This script accepts 2 parameters: the project name (mandatory) and "
+          "the refresh period in seconds (optional).")
+
+projName = sys.argv[1]
+if n == 3:
+    wait = int(sys.argv[2])
+else:
+    wait = 60
+
+path = pw.join('gui', 'no-tkinter')
+sys.path.insert(1, path)
+
+# Create a new project
+manager = Manager()
+
+if not manager.hasProject(projName):
+    usage("There is no project with this name: %s"
+          % pwutils.red(projName))
+
+# the project may be a soft link which may be unavailable to the cluster so get the real path
+try:
+    projectPath = os.readlink(manager.getProjectPath(projName))
+except:
+    projectPath = manager.getProjectPath(projName)
+
+project = Project(pw.Config.getDomain(), projectPath)
+project.load()
+while True:
+    runs = project.getRuns()
+    sleep(wait)

pyworkflow/project/scripts/schedule.py

@@ -0,0 +1,111 @@
+# **************************************************************************
+# *
+# * Authors:     Pablo Conesa (pconesa@cnb.csic.es)
+# *
+# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia, CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+
+import sys
+import os
+import time
+
+import pyworkflow as pw
+from pyworkflow.project import Manager
+from pyworkflow.project import Project
+import pyworkflow.utils as pwutils
+from pyworkflow.protocol import INITIAL_SLEEP_TIME
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+    
+    Usage: scipion python -m pyworkflow.project.scripts.schedule projectName 
+    
+              options: --ignore ProtClassName1 ProtClassName2 ProtClassLabel1 ...
+              
+        This script will schedule all the protocols in a project. If '--ignore' 
+          option is passed, it doesn't schedule those protocols that belongs to 
+          ProtClassName1 or ProtClassName2 class, also those protocols with a 
+          objLabel equals to ProtClassLabel1
+    """ % error)
+    sys.exit(1)
+
+
+n = len(sys.argv)
+
+if n < 2:
+    usage("This script accepts 1 mandatory parameter: the project name.")
+elif n > 2 and sys.argv[2] != '--ignore':
+    usage("The protocol class names to be ignored must be after a '--ignore' flag.")
+
+projName = sys.argv[1]
+
+# This fails, since it is triggering matplotlib.pyplot and then import error happens:
+# ... pyworkflow/gui/no-tkinter/_tkinter.py: invalid ELF header. If we want this back we might need to
+# invest some time "faking" tkinter again for python3.
+# path = pw.join('gui', 'no-tkinter')
+# sys.path.insert(1, path)
+
+# Create a new project
+manager = Manager()
+
+if not manager.hasProject(projName):
+    usage("There is no project with this name: %s"
+          % pwutils.red(projName))
+
+# the project may be a soft link which may be unavailable to the cluster
+# so get the real path
+
+try:
+    projectPath = os.readlink(manager.getProjectPath(projName))
+except:
+    projectPath = manager.getProjectPath(projName)
+
+project = Project(pw.Config.getDomain(), projectPath)
+project.load()
+
+runGraph = project.getRunsGraph()
+roots = runGraph.getRootNodes()
+
+# Now assuming that there is no dependencies between runs
+# and the graph is lineal
+
+for root in roots:
+    for child in root.getChilds():
+        workflow, _ = project._getSubworkflow(child.run)
+        for prot, level in workflow.values():
+            protLabelName = prot.getObjLabel()
+            if prot.forceSchedule.get():
+                print(pwutils.blueStr("\nStopping scheduling at '%s'. Force scheduling found.\n"
+                                      % protLabelName))
+                sys.exit()
+
+            protClassName = prot.getClassName()
+            if (protClassName not in sys.argv[3:] and
+                    protLabelName not in sys.argv[3:]):
+                project.scheduleProtocol(prot,
+                                         initialSleepTime=level*INITIAL_SLEEP_TIME)
+            else:
+                print(pwutils.blueStr("\nNot scheduling '%s' protocol named '%s'.\n"
+                                      % (protClassName, protLabelName)))

pyworkflow/project/scripts/stack2volume.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+
+import sys
+from glob import glob
+
+from pwem.convert import Ccp4Header
+import pyworkflow.utils as pwutils
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+    
+    Usage: scipion python -m pyworkflow.project.scripts.stack2volume PATH
+        PATH: path to look for stack files
+    The script will swap the dimensions in the header of a stack to make them 
+    volumes. Something like 10 x 1 x 10 x 10 will be changed to 1 x 10 x 10 x 10
+    """ % error)
+    sys.exit(1)    
+
+
+if len(sys.argv) != 2:
+    usage("Incorrect number of input parameters")
+
+path = sys.argv[1]
+
+print("Looking for files like: %s" % path)
+
+
+for file in glob(path):
+
+    print("Changing header of %s" % file)
+    try:
+        header = Ccp4Header(file, readHeader=True)
+        # Flag it as volume.
+        header.setISPG(401)
+        header.writeHeader()
+
+    except Exception as e:
+        print(pwutils.red("Failed to change header: %s" % e))