scipion-pyworkflow 3.11.0__py3-none-any.whl → 3.11.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyworkflow/apps/__init__.py +29 -0
- pyworkflow/apps/pw_manager.py +37 -0
- pyworkflow/apps/pw_plot.py +51 -0
- pyworkflow/apps/pw_project.py +113 -0
- pyworkflow/apps/pw_protocol_list.py +143 -0
- pyworkflow/apps/pw_protocol_run.py +51 -0
- pyworkflow/apps/pw_run_tests.py +267 -0
- pyworkflow/apps/pw_schedule_run.py +322 -0
- pyworkflow/apps/pw_sleep.py +37 -0
- pyworkflow/apps/pw_sync_data.py +439 -0
- pyworkflow/apps/pw_viewer.py +78 -0
- pyworkflow/constants.py +1 -1
- pyworkflow/gui/__init__.py +36 -0
- pyworkflow/gui/browser.py +760 -0
- pyworkflow/gui/canvas.py +1190 -0
- pyworkflow/gui/dialog.py +979 -0
- pyworkflow/gui/form.py +2726 -0
- pyworkflow/gui/graph.py +247 -0
- pyworkflow/gui/graph_layout.py +271 -0
- pyworkflow/gui/gui.py +566 -0
- pyworkflow/gui/matplotlib_image.py +233 -0
- pyworkflow/gui/plotter.py +247 -0
- pyworkflow/gui/project/__init__.py +25 -0
- pyworkflow/gui/project/base.py +192 -0
- pyworkflow/gui/project/constants.py +139 -0
- pyworkflow/gui/project/labels.py +205 -0
- pyworkflow/gui/project/project.py +491 -0
- pyworkflow/gui/project/searchprotocol.py +238 -0
- pyworkflow/gui/project/searchrun.py +181 -0
- pyworkflow/gui/project/steps.py +171 -0
- pyworkflow/gui/project/utils.py +332 -0
- pyworkflow/gui/project/variables.py +179 -0
- pyworkflow/gui/project/viewdata.py +472 -0
- pyworkflow/gui/project/viewprojects.py +510 -0
- pyworkflow/gui/project/viewprotocols.py +2116 -0
- pyworkflow/gui/project/viewprotocols_extra.py +562 -0
- pyworkflow/gui/text.py +771 -0
- pyworkflow/gui/tooltip.py +185 -0
- pyworkflow/gui/tree.py +684 -0
- pyworkflow/gui/widgets.py +307 -0
- pyworkflow/mapper/__init__.py +26 -0
- pyworkflow/mapper/mapper.py +222 -0
- pyworkflow/mapper/sqlite.py +1581 -0
- pyworkflow/mapper/sqlite_db.py +145 -0
- pyworkflow/project/__init__.py +31 -0
- pyworkflow/project/config.py +454 -0
- pyworkflow/project/manager.py +180 -0
- pyworkflow/project/project.py +2095 -0
- pyworkflow/project/usage.py +165 -0
- pyworkflow/protocol/__init__.py +38 -0
- pyworkflow/protocol/bibtex.py +48 -0
- pyworkflow/protocol/constants.py +87 -0
- pyworkflow/protocol/executor.py +483 -0
- pyworkflow/protocol/hosts.py +317 -0
- pyworkflow/protocol/launch.py +277 -0
- pyworkflow/protocol/package.py +42 -0
- pyworkflow/protocol/params.py +781 -0
- pyworkflow/protocol/protocol.py +2707 -0
- pyworkflow/tests/__init__.py +29 -0
- pyworkflow/tests/test_utils.py +25 -0
- pyworkflow/tests/tests.py +341 -0
- pyworkflow/utils/__init__.py +38 -0
- pyworkflow/utils/dataset.py +414 -0
- pyworkflow/utils/echo.py +104 -0
- pyworkflow/utils/graph.py +169 -0
- pyworkflow/utils/log.py +293 -0
- pyworkflow/utils/path.py +528 -0
- pyworkflow/utils/process.py +153 -0
- pyworkflow/utils/profiler.py +92 -0
- pyworkflow/utils/progressbar.py +154 -0
- pyworkflow/utils/properties.py +617 -0
- pyworkflow/utils/reflection.py +129 -0
- pyworkflow/utils/utils.py +880 -0
- pyworkflow/utils/which.py +229 -0
- pyworkflow/webservices/__init__.py +8 -0
- pyworkflow/webservices/config.py +8 -0
- pyworkflow/webservices/notifier.py +152 -0
- pyworkflow/webservices/repository.py +59 -0
- pyworkflow/webservices/workflowhub.py +74 -0
- pyworkflowtests/tests/__init__.py +0 -0
- pyworkflowtests/tests/test_canvas.py +72 -0
- pyworkflowtests/tests/test_domain.py +45 -0
- pyworkflowtests/tests/test_logs.py +74 -0
- pyworkflowtests/tests/test_mappers.py +392 -0
- pyworkflowtests/tests/test_object.py +507 -0
- pyworkflowtests/tests/test_project.py +42 -0
- pyworkflowtests/tests/test_protocol_execution.py +146 -0
- pyworkflowtests/tests/test_protocol_export.py +78 -0
- pyworkflowtests/tests/test_protocol_output.py +158 -0
- pyworkflowtests/tests/test_streaming.py +47 -0
- pyworkflowtests/tests/test_utils.py +210 -0
- {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/METADATA +2 -2
- scipion_pyworkflow-3.11.1.dist-info/RECORD +161 -0
- scipion_pyworkflow-3.11.0.dist-info/RECORD +0 -71
- {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/WHEEL +0 -0
- {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/entry_points.txt +0 -0
- {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/licenses/LICENSE.txt +0 -0
- {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,29 @@
|
|
1
|
+
# **************************************************************************
|
2
|
+
# *
|
3
|
+
# * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
|
4
|
+
# *
|
5
|
+
# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
|
6
|
+
# *
|
7
|
+
# * This program is free software; you can redistribute it and/or modify
|
8
|
+
# * it under the terms of the GNU General Public License as published by
|
9
|
+
# * the Free Software Foundation; either version 3 of the License, or
|
10
|
+
# * (at your option) any later version.
|
11
|
+
# *
|
12
|
+
# * This program is distributed in the hope that it will be useful,
|
13
|
+
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# * GNU General Public License for more details.
|
16
|
+
# *
|
17
|
+
# * You should have received a copy of the GNU General Public License
|
18
|
+
# * along with this program; if not, write to the Free Software
|
19
|
+
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
20
|
+
# * 02111-1307 USA
|
21
|
+
# *
|
22
|
+
# * All comments concerning this program package may be sent to the
|
23
|
+
# * e-mail address 'scipion@cnb.csic.es'
|
24
|
+
# *
|
25
|
+
# **************************************************************************
|
26
|
+
"""
|
27
|
+
Configuration sub-package
|
28
|
+
"""
|
29
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
|
3
|
+
# **************************************************************************
|
4
|
+
# *
|
5
|
+
# * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
|
6
|
+
# *
|
7
|
+
# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
|
8
|
+
# *
|
9
|
+
# * This program is free software; you can redistribute it and/or modify
|
10
|
+
# * it under the terms of the GNU General Public License as published by
|
11
|
+
# * the Free Software Foundation; either version 3 of the License, or
|
12
|
+
# * (at your option) any later version.
|
13
|
+
# *
|
14
|
+
# * This program is distributed in the hope that it will be useful,
|
15
|
+
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
+
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
+
# * GNU General Public License for more details.
|
18
|
+
# *
|
19
|
+
# * You should have received a copy of the GNU General Public License
|
20
|
+
# * along with this program; if not, write to the Free Software
|
21
|
+
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
22
|
+
# * 02111-1307 USA
|
23
|
+
# *
|
24
|
+
# * All comments concerning this program package may be sent to the
|
25
|
+
# * e-mail address 'scipion@cnb.csic.es'
|
26
|
+
# *
|
27
|
+
# **************************************************************************
|
28
|
+
"""
|
29
|
+
Launch the windows with all Projects.
|
30
|
+
"""
|
31
|
+
|
32
|
+
from pyworkflow.gui.project import ProjectManagerWindow
|
33
|
+
|
34
|
+
|
35
|
+
if __name__ == '__main__':
|
36
|
+
|
37
|
+
ProjectManagerWindow().show()
|
@@ -0,0 +1,51 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
|
3
|
+
|
4
|
+
import argparse
|
5
|
+
from pwem.viewers import EmPlotter
|
6
|
+
from pyworkflow.gui.plotter import Plotter
|
7
|
+
|
8
|
+
|
9
|
+
def main():
|
10
|
+
parser = argparse.ArgumentParser(prog='Scipion Plot')
|
11
|
+
parser.add_argument('--file', help='File to visualize', required=True)
|
12
|
+
parser.add_argument('--block', help='Block to visualize')
|
13
|
+
parser.add_argument('--type', help='Plot type')
|
14
|
+
parser.add_argument('--columns', help='Columns to plot')
|
15
|
+
parser.add_argument('--xcolumn', help='X Column to plot')
|
16
|
+
parser.add_argument('--orderColumn', help='Column to order')
|
17
|
+
parser.add_argument('--orderDir', help='Order direction(ASC, DESC)')
|
18
|
+
parser.add_argument('--bins', help='If plot type is histogram, number of bins')
|
19
|
+
parser.add_argument('--colors', help='Colors to plot columns')
|
20
|
+
parser.add_argument('--styles', help='Styles to plot columns')
|
21
|
+
parser.add_argument('--markers', help='Markers to plot columns')
|
22
|
+
parser.add_argument('--title', help='Plot title', default='')
|
23
|
+
parser.add_argument('--ytitle', help='Y axis title', default='')
|
24
|
+
parser.add_argument('--xtitle', help='X axis title', default='')
|
25
|
+
|
26
|
+
args = parser.parse_args()
|
27
|
+
plotfile = args.file
|
28
|
+
block = args.block if args.block else ''
|
29
|
+
type = args.type
|
30
|
+
columns = args.columns
|
31
|
+
xcolumn = args.xcolumn
|
32
|
+
orderColumn = args.orderColumn
|
33
|
+
orderDir = args.orderDir
|
34
|
+
|
35
|
+
bins = args.bins
|
36
|
+
colors = args.colors
|
37
|
+
styles = args.styles
|
38
|
+
markers = args.markers
|
39
|
+
title = args.title
|
40
|
+
xtitle = args.xtitle
|
41
|
+
ytitle = args.ytitle
|
42
|
+
|
43
|
+
Plotter.setBackend('TkAgg')
|
44
|
+
plotter = EmPlotter.createFromFile(
|
45
|
+
plotfile, block, type, columns, colors, styles, markers, xcolumn,
|
46
|
+
ytitle, xtitle, title, bins, orderColumn, orderDir)
|
47
|
+
plotter.show(block=True)
|
48
|
+
|
49
|
+
|
50
|
+
if __name__ == '__main__':
|
51
|
+
main()
|
@@ -0,0 +1,113 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# **************************************************************************
|
3
|
+
# *
|
4
|
+
# * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
|
5
|
+
# *
|
6
|
+
# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
|
7
|
+
# *
|
8
|
+
# * This program is free software; you can redistribute it and/or modify
|
9
|
+
# * it under the terms of the GNU General Public License as published by
|
10
|
+
# * the Free Software Foundation; either version 3 of the License, or
|
11
|
+
# * (at your option) any later version.
|
12
|
+
# *
|
13
|
+
# * This program is distributed in the hope that it will be useful,
|
14
|
+
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
15
|
+
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
16
|
+
# * GNU General Public License for more details.
|
17
|
+
# *
|
18
|
+
# * You should have received a copy of the GNU General Public License
|
19
|
+
# * along with this program; if not, write to the Free Software
|
20
|
+
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
21
|
+
# * 02111-1307 USA
|
22
|
+
# *
|
23
|
+
# * All comments concerning this program package may be sent to the
|
24
|
+
# * e-mail address 'scipion@cnb.csic.es'
|
25
|
+
# *
|
26
|
+
# **************************************************************************
|
27
|
+
"""
|
28
|
+
Launch main project window
|
29
|
+
"""
|
30
|
+
|
31
|
+
import sys
|
32
|
+
import os
|
33
|
+
|
34
|
+
from pyworkflow import Config
|
35
|
+
from pyworkflow.project import Manager
|
36
|
+
from pyworkflow.gui.project import ProjectWindow
|
37
|
+
import pyworkflow.utils as pwutils
|
38
|
+
|
39
|
+
HERE = 'here'
|
40
|
+
LAST = 'last'
|
41
|
+
LIST = 'list'
|
42
|
+
|
43
|
+
def openProject(projectName):
|
44
|
+
""" Opens a scipion project:
|
45
|
+
|
46
|
+
:param projectName: Name of an existing project to open,
|
47
|
+
or "here" to create a project in the current working dir,
|
48
|
+
or "last" to open the most recent project
|
49
|
+
|
50
|
+
"""
|
51
|
+
manager = Manager()
|
52
|
+
projName = os.path.basename(projectName)
|
53
|
+
|
54
|
+
|
55
|
+
if projName == LIST:
|
56
|
+
showProjectList(manager)
|
57
|
+
return
|
58
|
+
# Handle special name 'here' to create a project
|
59
|
+
# from the current directory
|
60
|
+
elif projName == HERE:
|
61
|
+
cwd = Config.SCIPION_CWD
|
62
|
+
|
63
|
+
if " " in cwd:
|
64
|
+
print("Projects can't have spaces in the name: %s" % cwd)
|
65
|
+
sys.exit(1)
|
66
|
+
|
67
|
+
print("\nYou are trying to create a project here:",
|
68
|
+
pwutils.cyan(cwd))
|
69
|
+
|
70
|
+
if os.listdir(cwd):
|
71
|
+
print(pwutils.red('\nWARNING: this folder is not empty!!!'))
|
72
|
+
key = input("\nDo you want to create a project here? [y/N]?")
|
73
|
+
|
74
|
+
if key.lower().strip() != 'y':
|
75
|
+
print("\nAborting...")
|
76
|
+
sys.exit(0)
|
77
|
+
else:
|
78
|
+
print("\nCreating project....")
|
79
|
+
projName = os.path.basename(cwd)
|
80
|
+
projDir = os.path.dirname(cwd)
|
81
|
+
manager.createProject(projName, location=projDir)
|
82
|
+
|
83
|
+
elif projName == LAST: # Get last project
|
84
|
+
projects = manager.listProjects()
|
85
|
+
if not projects:
|
86
|
+
sys.exit("No projects yet, cannot open the last one.")
|
87
|
+
projName = projects[0].projName
|
88
|
+
|
89
|
+
projPath = manager.getProjectPath(projName)
|
90
|
+
|
91
|
+
if os.path.exists(projPath):
|
92
|
+
projWindow = ProjectWindow(projPath)
|
93
|
+
projWindow.show()
|
94
|
+
else:
|
95
|
+
print("Can't open project %s. It does not exist" % projPath)
|
96
|
+
|
97
|
+
#Show the list of projects
|
98
|
+
showProjectList(manager)
|
99
|
+
|
100
|
+
def showProjectList(manager):
|
101
|
+
|
102
|
+
projects = manager.listProjects()
|
103
|
+
|
104
|
+
print("\n******** LIST OF PROJECTS *******\n")
|
105
|
+
for project in projects:
|
106
|
+
print(project.projName)
|
107
|
+
print("\n")
|
108
|
+
if __name__ == '__main__':
|
109
|
+
|
110
|
+
if len(sys.argv) > 1:
|
111
|
+
openProject(sys.argv[1])
|
112
|
+
else:
|
113
|
+
print("usage: pw_project.py PROJECT_NAME or %s or %s or %s" % (HERE, LAST, LIST))
|
@@ -0,0 +1,143 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# **************************************************************************
|
3
|
+
# *
|
4
|
+
# * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
|
5
|
+
# *
|
6
|
+
# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
|
7
|
+
# *
|
8
|
+
# * This program is free software; you can redistribute it and/or modify
|
9
|
+
# * it under the terms of the GNU General Public License as published by
|
10
|
+
# * the Free Software Foundation; either version 3 of the License, or
|
11
|
+
# * (at your option) any later version.
|
12
|
+
# *
|
13
|
+
# * This program is distributed in the hope that it will be useful,
|
14
|
+
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
15
|
+
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
16
|
+
# * GNU General Public License for more details.
|
17
|
+
# *
|
18
|
+
# * You should have received a copy of the GNU General Public License
|
19
|
+
# * along with this program; if not, write to the Free Software
|
20
|
+
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
21
|
+
# * 02111-1307 USA
|
22
|
+
# *
|
23
|
+
# * All comments concerning this program package may be sent to the
|
24
|
+
# * e-mail address 'scipion@cnb.csic.es'
|
25
|
+
# *
|
26
|
+
# **************************************************************************
|
27
|
+
###
|
28
|
+
"""
|
29
|
+
List all existing protocols within Scipion
|
30
|
+
"""
|
31
|
+
|
32
|
+
import sys
|
33
|
+
|
34
|
+
from pwem.protocols import (ProtImport, ProtMicrographs, ProtParticles, Prot2D,
|
35
|
+
Prot3D)
|
36
|
+
from pyworkflow import Config
|
37
|
+
from pyworkflow.viewer import Viewer
|
38
|
+
from pyworkflow.protocol.protocol import Protocol
|
39
|
+
|
40
|
+
|
41
|
+
def getFirstLine(doc):
|
42
|
+
""" Get the first non empty line from doc. """
|
43
|
+
if doc:
|
44
|
+
for lines in doc.split('\n'):
|
45
|
+
l = lines.strip()
|
46
|
+
if l:
|
47
|
+
return l
|
48
|
+
return ''
|
49
|
+
|
50
|
+
|
51
|
+
def hasDoubleInheritance(classRef):
|
52
|
+
# loop while class has single parent
|
53
|
+
numParents = len(classRef.__bases__)
|
54
|
+
while numParents == 1 and classRef is not Protocol:
|
55
|
+
classRef = classRef.__bases__[0]
|
56
|
+
numParents = len(classRef.__bases__)
|
57
|
+
if numParents > 1:
|
58
|
+
return True
|
59
|
+
else:
|
60
|
+
return False
|
61
|
+
|
62
|
+
|
63
|
+
if __name__ == '__main__':
|
64
|
+
count = 0
|
65
|
+
withDoc = '--with-doc' in sys.argv
|
66
|
+
extended = '--extended' in sys.argv
|
67
|
+
ai = '--ai' in sys.argv
|
68
|
+
|
69
|
+
emProtocolsDict = Config.getDomain().getProtocols()
|
70
|
+
|
71
|
+
protDict = {}
|
72
|
+
|
73
|
+
# Group protocols by package name
|
74
|
+
for k, v in emProtocolsDict.items():
|
75
|
+
packageName = v.getClassPackageName()
|
76
|
+
|
77
|
+
if packageName not in protDict:
|
78
|
+
protDict[packageName] = []
|
79
|
+
|
80
|
+
if not issubclass(v, Viewer) and not v.isBase():
|
81
|
+
if extended:
|
82
|
+
protTuple = (k, v, hasDoubleInheritance(v),
|
83
|
+
v().allowMpi, v().numberOfMpi,
|
84
|
+
v().allowThreads, v().numberOfThreads,
|
85
|
+
v().stepsExecutionMode)
|
86
|
+
else:
|
87
|
+
protTuple = (k, v)
|
88
|
+
protDict[packageName].append(protTuple)
|
89
|
+
|
90
|
+
def iterGroups(protDict):
|
91
|
+
groups = list(protDict.keys())
|
92
|
+
groups.sort(key=lambda x: 1000-len(protDict[x]))
|
93
|
+
|
94
|
+
for g in groups:
|
95
|
+
yield g, protDict[g]
|
96
|
+
|
97
|
+
def printProtocols(prots):
|
98
|
+
protList = [(p[0], p[1], p[1].getClassLabel()) for p in prots]
|
99
|
+
protList.sort(key=lambda x: x[2])
|
100
|
+
|
101
|
+
for k, v, l in protList:
|
102
|
+
doc = getFirstLine(v.__doc__) if withDoc else ''
|
103
|
+
print("* link:%s[%s]: %s" % (k, l, doc))
|
104
|
+
|
105
|
+
|
106
|
+
if withDoc:
|
107
|
+
for group, prots in iterGroups(protDict):
|
108
|
+
print("Package: ", group, "(%d protocols)" % len(prots))
|
109
|
+
for p in prots:
|
110
|
+
print(" %s ( %s ):" % (p[1].getClassLabel(), p[0]))
|
111
|
+
print(" ", p[1].__doc__)
|
112
|
+
print("-" * 100)
|
113
|
+
|
114
|
+
else:
|
115
|
+
if extended:
|
116
|
+
formatStr = "{:<15}\t{:<35}\t{:<35}" + "\t{:<20}" * 6
|
117
|
+
print(formatStr.format("PACKAGE", "PROTOCOL",
|
118
|
+
"LABEL", "DOUBLE_INHERITANCE",
|
119
|
+
"ALLOWS_MPI", "NUM_MPI",
|
120
|
+
"ALLOWS_THREADS", "NUM_THREADS",
|
121
|
+
"STEPS_EXEC_MODE"))
|
122
|
+
for group, prots in iterGroups(protDict):
|
123
|
+
for p in prots:
|
124
|
+
print(formatStr.format(group, p[0],
|
125
|
+
p[1].getClassLabel(), *p[2:]))
|
126
|
+
elif ai:
|
127
|
+
for group, prots in iterGroups(protDict):
|
128
|
+
for protClassName, protClass in prots:
|
129
|
+
print("\nThe protocol named \"%s\" can be found in the %s plugin." %(protClass._label, group))
|
130
|
+
print("\"%s\" protocol help is as follows:\n %s." % (protClass._label, protClass.__doc__))
|
131
|
+
instance = protClass()
|
132
|
+
|
133
|
+
for name, param in instance._definition.iterParams():
|
134
|
+
print("\"%s\" has a \"%s\" parameter that is explained as: %s" % (protClass._label, param.label, param.help))
|
135
|
+
|
136
|
+
|
137
|
+
|
138
|
+
else:
|
139
|
+
formatStr = "{:<15}\t{:<35}\t{:<35}"
|
140
|
+
print(formatStr.format("PACKAGE", "PROTOCOL", "LABEL"))
|
141
|
+
for group, prots in iterGroups(protDict):
|
142
|
+
for k, v in prots:
|
143
|
+
print(formatStr.format(group, k, v.getClassLabel()))
|
@@ -0,0 +1,51 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# **************************************************************************
|
3
|
+
# *
|
4
|
+
# * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
|
5
|
+
# *
|
6
|
+
# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
|
7
|
+
# *
|
8
|
+
# * This program is free software; you can redistribute it and/or modify
|
9
|
+
# * it under the terms of the GNU General Public License as published by
|
10
|
+
# * the Free Software Foundation; either version 3 of the License, or
|
11
|
+
# * (at your option) any later version.
|
12
|
+
# *
|
13
|
+
# * This program is distributed in the hope that it will be useful,
|
14
|
+
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
15
|
+
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
16
|
+
# * GNU General Public License for more details.
|
17
|
+
# *
|
18
|
+
# * You should have received a copy of the GNU General Public License
|
19
|
+
# * along with this program; if not, write to the Free Software
|
20
|
+
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
21
|
+
# * 02111-1307 USA
|
22
|
+
# *
|
23
|
+
# * All comments concerning this program package may be sent to the
|
24
|
+
# * e-mail address 'scipion@cnb.csic.es'
|
25
|
+
# *
|
26
|
+
# **************************************************************************
|
27
|
+
"""
|
28
|
+
This module is responsible for launching protocol executions.
|
29
|
+
"""
|
30
|
+
import sys
|
31
|
+
from pyworkflow.utils import LoggingConfigurator
|
32
|
+
|
33
|
+
if __name__ == '__main__':
|
34
|
+
|
35
|
+
if len(sys.argv) == 6:
|
36
|
+
projPath = sys.argv[1]
|
37
|
+
dbPath = sys.argv[2]
|
38
|
+
protId = int(sys.argv[3])
|
39
|
+
stdOut = sys.argv[4]
|
40
|
+
stdErr = sys.argv[5]
|
41
|
+
|
42
|
+
LoggingConfigurator.setUpProtocolRunLogging(stdOut, stdErr)
|
43
|
+
|
44
|
+
from pyworkflow.protocol import runProtocolMain
|
45
|
+
runProtocolMain(projPath, dbPath, protId)
|
46
|
+
|
47
|
+
else:
|
48
|
+
from os.path import basename
|
49
|
+
print("usage: %s projPath dbPath protocolID pathToStdoutLog pathToStdErrLog." % basename(sys.argv[0]))
|
50
|
+
print("sys.argv: %s" % sys.argv)
|
51
|
+
sys.exit(1)
|
@@ -0,0 +1,267 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# **************************************************************************
|
3
|
+
# *
|
4
|
+
# * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
|
5
|
+
# *
|
6
|
+
# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
|
7
|
+
# *
|
8
|
+
# * This program is free software; you can redistribute it and/or modify
|
9
|
+
# * it under the terms of the GNU General Public License as published by
|
10
|
+
# * the Free Software Foundation; either version 3 of the License, or
|
11
|
+
# * (at your option) any later version.
|
12
|
+
# *
|
13
|
+
# * This program is distributed in the hope that it will be useful,
|
14
|
+
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
15
|
+
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
16
|
+
# * GNU General Public License for more details.
|
17
|
+
# *
|
18
|
+
# * You should have received a copy of the GNU General Public License
|
19
|
+
# * along with this program; if not, write to the Free Software
|
20
|
+
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
21
|
+
# * 02111-1307 USA
|
22
|
+
# *
|
23
|
+
# * All comments concerning this program package may be sent to the
|
24
|
+
# * e-mail address 'scipion@cnb.csic.es'
|
25
|
+
# *
|
26
|
+
# **************************************************************************
|
27
|
+
|
28
|
+
"""
|
29
|
+
Run or show the selected tests. Tests can be selected by giving
|
30
|
+
the "case", or by giving the paths and file pattern to use for
|
31
|
+
searching them.
|
32
|
+
"""
|
33
|
+
from pyworkflow.utils import LoggingConfigurator
|
34
|
+
import argparse
|
35
|
+
from collections import OrderedDict
|
36
|
+
|
37
|
+
import pyworkflow.tests as pwtests
|
38
|
+
from pyworkflow import getTestsScript, SCIPION_TESTS_CMD, Config
|
39
|
+
|
40
|
+
from pyworkflow.tests import *
|
41
|
+
|
42
|
+
MODULE = 0
|
43
|
+
CLASS = 1
|
44
|
+
TEST = 2
|
45
|
+
|
46
|
+
|
47
|
+
class Tester:
|
48
|
+
def main(self, args=None):
|
49
|
+
|
50
|
+
# Trigger plugin's variable definition
|
51
|
+
Config.getDomain().getPlugins()
|
52
|
+
|
53
|
+
parser = argparse.ArgumentParser(prog=self.getTestsCommand(), description=__doc__)
|
54
|
+
g = parser.add_mutually_exclusive_group()
|
55
|
+
g.add_argument('--run', action='store_true', help='run the selected tests')
|
56
|
+
g.add_argument('--show', action='store_true', help='show available tests',
|
57
|
+
default=True)
|
58
|
+
|
59
|
+
add = parser.add_argument # shortcut
|
60
|
+
|
61
|
+
add('--pattern', default='test*.py',
|
62
|
+
help='pattern for the files that will be used in the tests')
|
63
|
+
add('--grep', default=None, nargs='+',
|
64
|
+
help='only show/run tests containing the provided words')
|
65
|
+
add('--skip', default=None, nargs='+',
|
66
|
+
help='skip tests that contains these words')
|
67
|
+
add('--log', default=None, nargs='?',
|
68
|
+
help="Generate logs files with the output of each test.")
|
69
|
+
add('--mode', default='classes', choices=['modules', 'classes', 'onlyclasses', 'all'],
|
70
|
+
help='how much detail to give in show mode')
|
71
|
+
add('tests', metavar='TEST', nargs='*',
|
72
|
+
help='test case from string identifier (module, class or callable)')
|
73
|
+
args = parser.parse_args(args)
|
74
|
+
|
75
|
+
if not args.run and not args.show and not args.tests:
|
76
|
+
sys.exit(parser.format_help())
|
77
|
+
|
78
|
+
# Logging stuff first
|
79
|
+
self.log = args.log
|
80
|
+
|
81
|
+
if self.log:
|
82
|
+
LoggingConfigurator.setupLogging(logFile=self.log)
|
83
|
+
else:
|
84
|
+
logging.basicConfig(level=Config.SCIPION_LOG_LEVEL, format=Config.SCIPION_LOG_FORMAT)
|
85
|
+
|
86
|
+
self.logger = logging.getLogger(__name__)
|
87
|
+
|
88
|
+
# This goes intentionally to the output. Is not a logging line._S
|
89
|
+
print("Scanning tests...")
|
90
|
+
|
91
|
+
|
92
|
+
testsDict = OrderedDict()
|
93
|
+
testLoader = unittest.defaultTestLoader
|
94
|
+
|
95
|
+
if args.tests:
|
96
|
+
print(pwutils.cyanStr("Loading test/s %s" % args.tests))
|
97
|
+
|
98
|
+
# In this case the tests are passed as argument.
|
99
|
+
# The full name of the test will be used to load it.
|
100
|
+
testsDict['tests'] = unittest.TestSuite()
|
101
|
+
for t in args.tests:
|
102
|
+
try:
|
103
|
+
testsDict['tests'].addTests(testLoader.loadTestsFromName(t))
|
104
|
+
except Exception as e:
|
105
|
+
self.logger.error('Cannot load test %s -- skipping' % t, exc_info=True)
|
106
|
+
else:
|
107
|
+
# In this other case, we will load the test available
|
108
|
+
# from pyworkflow and the other plugins
|
109
|
+
# self.paths = [('pyworkflow', os.path.dirname(os.path.dirname(pw.__file__)))]
|
110
|
+
self.paths = []
|
111
|
+
for name, plugin in pw.Config.getDomain().getPlugins().items():
|
112
|
+
self.paths.append((name, os.path.dirname(plugin.__path__[0])))
|
113
|
+
for k, p in self.paths:
|
114
|
+
testPath = os.path.join(p, k, 'tests')
|
115
|
+
if os.path.exists(testPath):
|
116
|
+
self.logger.debug("Discovering tests at %s" % testPath)
|
117
|
+
testsDict[k] = testLoader.discover(testPath,
|
118
|
+
pattern=args.pattern,
|
119
|
+
top_level_dir=p)
|
120
|
+
|
121
|
+
self.grep = [g.lower() for g in args.grep] if args.grep else []
|
122
|
+
self.skip = [s.lower() for s in args.skip] if args.skip else []
|
123
|
+
self.mode = args.mode
|
124
|
+
|
125
|
+
if args.tests:
|
126
|
+
self.runSingleTest(testsDict['tests'])
|
127
|
+
|
128
|
+
elif args.run:
|
129
|
+
for moduleName, tests in testsDict.items():
|
130
|
+
self.logger.info(pwutils.cyanStr(">>>> %s" % moduleName))
|
131
|
+
self.runTests(moduleName, tests)
|
132
|
+
|
133
|
+
elif args.grep:
|
134
|
+
pattern = args.grep[0]
|
135
|
+
for moduleName, tests in testsDict.items():
|
136
|
+
self.printTests(pattern, tests)
|
137
|
+
|
138
|
+
else:
|
139
|
+
for moduleName, tests in testsDict.items():
|
140
|
+
if self._match(moduleName):
|
141
|
+
print(pwutils.cyanStr(">>>> %s" % moduleName))
|
142
|
+
self.printTests(moduleName, tests)
|
143
|
+
|
144
|
+
def _match(self, itemName):
|
145
|
+
# Add a space to allow " tomo." as a filter for example to narrow search to tomo and leaving out xmipptomo, emntomo, ...
|
146
|
+
itemLower = " " + itemName.lower()
|
147
|
+
grep = (not self.grep or
|
148
|
+
all(g in itemLower for g in self.grep))
|
149
|
+
skip = (self.skip and
|
150
|
+
any(s in itemLower for s in self.skip))
|
151
|
+
|
152
|
+
return grep and not skip
|
153
|
+
|
154
|
+
def __iterTests(self, test):
|
155
|
+
""" Recursively iterate over a testsuite. """
|
156
|
+
self.logger.debug("__iterTests: %s, is-suite: %s" % (str(test.__class__),
|
157
|
+
isinstance(test, unittest.TestSuite)))
|
158
|
+
|
159
|
+
if isinstance(test, unittest.TestSuite):
|
160
|
+
for t in test:
|
161
|
+
self.__iterTests(t)
|
162
|
+
else:
|
163
|
+
yield test
|
164
|
+
|
165
|
+
def _visitTests(self, moduleName, tests, newItemCallback):
|
166
|
+
""" Show the list of tests available """
|
167
|
+
mode = self.mode
|
168
|
+
|
169
|
+
assert mode in ['modules', 'classes', 'onlyclasses', 'all'], 'Unknown mode %s' % mode
|
170
|
+
|
171
|
+
# First flatten the list of tests.
|
172
|
+
# testsFlat = list(iter(self.__iterTests(tests)))
|
173
|
+
|
174
|
+
testsFlat = []
|
175
|
+
toCheck = [t for t in tests]
|
176
|
+
|
177
|
+
while toCheck:
|
178
|
+
test = toCheck.pop()
|
179
|
+
if isinstance(test, unittest.TestSuite):
|
180
|
+
toCheck += [t for t in test]
|
181
|
+
elif test not in testsFlat:
|
182
|
+
testsFlat.append(test)
|
183
|
+
|
184
|
+
# Follow the flattened list of tests and show the module, class
|
185
|
+
# and name, in a nice way.
|
186
|
+
lastClass = None
|
187
|
+
lastModule = None
|
188
|
+
if testsFlat:
|
189
|
+
for t in testsFlat:
|
190
|
+
|
191
|
+
testModuleName, className, testName = t.id().rsplit('.', 2)
|
192
|
+
|
193
|
+
# If there is a failure loading the test, show it
|
194
|
+
errorStr = 'unittest.loader._FailedTest.'
|
195
|
+
if testModuleName.startswith(errorStr):
|
196
|
+
newName = t.id().replace(errorStr, '')
|
197
|
+
if self._match(newName):
|
198
|
+
self.logger.error(
|
199
|
+
pwutils.redStr('Error loading the test. Please, run the test for more information: ' + newName))
|
200
|
+
continue
|
201
|
+
|
202
|
+
if testModuleName != lastModule:
|
203
|
+
lastModule = testModuleName
|
204
|
+
if mode != 'onlyclasses':
|
205
|
+
newItemCallback(MODULE, "%s" % testModuleName)
|
206
|
+
|
207
|
+
if mode in ['classes', 'onlyclasses', 'all'] and className != lastClass:
|
208
|
+
lastClass = className
|
209
|
+
newItemCallback(CLASS, "%s.%s"
|
210
|
+
% (testModuleName, className))
|
211
|
+
|
212
|
+
if mode == 'all':
|
213
|
+
newItemCallback(TEST, "%s.%s.%s"
|
214
|
+
% (testModuleName, className, testName))
|
215
|
+
else:
|
216
|
+
if not self.grep:
|
217
|
+
self.logger.warning(pwutils.greenStr(' The plugin does not have any test'))
|
218
|
+
|
219
|
+
def _printNewItem(self, itemType, itemName):
|
220
|
+
if self._match(itemName):
|
221
|
+
spaces = (itemType * 2) * ' '
|
222
|
+
# Do not use intentionally the logger. This is not a logging output but a GUI output
|
223
|
+
print("%s %s %s" % (spaces, self.getTestsCommand(), itemName))
|
224
|
+
|
225
|
+
def getTestsCommand(self):
|
226
|
+
return os.environ.get(SCIPION_TESTS_CMD, getTestsScript())
|
227
|
+
|
228
|
+
def printTests(self, moduleName, tests):
|
229
|
+
self._visitTests(moduleName, tests, self._printNewItem)
|
230
|
+
|
231
|
+
def _runNewItem(self, itemType, itemName):
|
232
|
+
if self._match(itemName):
|
233
|
+
spaces = (itemType * 2) * ' '
|
234
|
+
script = getTestsScript()
|
235
|
+
cmd = "%s %s %s %s" % (pw.PYTHON, script, spaces, itemName)
|
236
|
+
run = ((itemType == MODULE and self.mode == 'modules') or
|
237
|
+
(itemType == CLASS and self.mode in ('classes', 'onlyclasses')) or
|
238
|
+
(itemType == TEST and self.mode == 'all'))
|
239
|
+
if run:
|
240
|
+
if self.log:
|
241
|
+
# logFile = join(self.testsDir, '%s.txt' % itemName)
|
242
|
+
cmdFull = cmd + " >> %s 2>&1" % self.log
|
243
|
+
else:
|
244
|
+
logFile = ''
|
245
|
+
cmdFull = cmd
|
246
|
+
|
247
|
+
self.logger.info(pwutils.greenStr(cmdFull))
|
248
|
+
t = pwutils.Timer()
|
249
|
+
t.tic()
|
250
|
+
self.testCount += 1
|
251
|
+
os.system(cmdFull)
|
252
|
+
|
253
|
+
def runTests(self, moduleName, tests):
|
254
|
+
|
255
|
+
self.testCount = 0
|
256
|
+
self._visitTests(moduleName, tests, self._runNewItem)
|
257
|
+
|
258
|
+
def runSingleTest(self, tests):
|
259
|
+
result = pwtests.GTestResult()
|
260
|
+
tests.run(result)
|
261
|
+
result.doReport()
|
262
|
+
resultPassed = result.numberTests - result.testFailed
|
263
|
+
sys.exit(1 if result.testFailed > 0 or resultPassed == 0 else 0)
|
264
|
+
|
265
|
+
|
266
|
+
if __name__ == '__main__':
|
267
|
+
Tester().main()
|