scipion-pyworkflow 3.11.0__py3-none-any.whl → 3.11.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. pyworkflow/apps/__init__.py +29 -0
  2. pyworkflow/apps/pw_manager.py +37 -0
  3. pyworkflow/apps/pw_plot.py +51 -0
  4. pyworkflow/apps/pw_project.py +113 -0
  5. pyworkflow/apps/pw_protocol_list.py +143 -0
  6. pyworkflow/apps/pw_protocol_run.py +51 -0
  7. pyworkflow/apps/pw_run_tests.py +267 -0
  8. pyworkflow/apps/pw_schedule_run.py +322 -0
  9. pyworkflow/apps/pw_sleep.py +37 -0
  10. pyworkflow/apps/pw_sync_data.py +439 -0
  11. pyworkflow/apps/pw_viewer.py +78 -0
  12. pyworkflow/constants.py +1 -1
  13. pyworkflow/gui/__init__.py +36 -0
  14. pyworkflow/gui/browser.py +760 -0
  15. pyworkflow/gui/canvas.py +1190 -0
  16. pyworkflow/gui/dialog.py +979 -0
  17. pyworkflow/gui/form.py +2726 -0
  18. pyworkflow/gui/graph.py +247 -0
  19. pyworkflow/gui/graph_layout.py +271 -0
  20. pyworkflow/gui/gui.py +566 -0
  21. pyworkflow/gui/matplotlib_image.py +233 -0
  22. pyworkflow/gui/plotter.py +247 -0
  23. pyworkflow/gui/project/__init__.py +25 -0
  24. pyworkflow/gui/project/base.py +192 -0
  25. pyworkflow/gui/project/constants.py +139 -0
  26. pyworkflow/gui/project/labels.py +205 -0
  27. pyworkflow/gui/project/project.py +491 -0
  28. pyworkflow/gui/project/searchprotocol.py +238 -0
  29. pyworkflow/gui/project/searchrun.py +181 -0
  30. pyworkflow/gui/project/steps.py +171 -0
  31. pyworkflow/gui/project/utils.py +332 -0
  32. pyworkflow/gui/project/variables.py +179 -0
  33. pyworkflow/gui/project/viewdata.py +472 -0
  34. pyworkflow/gui/project/viewprojects.py +510 -0
  35. pyworkflow/gui/project/viewprotocols.py +2116 -0
  36. pyworkflow/gui/project/viewprotocols_extra.py +562 -0
  37. pyworkflow/gui/text.py +771 -0
  38. pyworkflow/gui/tooltip.py +185 -0
  39. pyworkflow/gui/tree.py +684 -0
  40. pyworkflow/gui/widgets.py +307 -0
  41. pyworkflow/mapper/__init__.py +26 -0
  42. pyworkflow/mapper/mapper.py +222 -0
  43. pyworkflow/mapper/sqlite.py +1581 -0
  44. pyworkflow/mapper/sqlite_db.py +145 -0
  45. pyworkflow/project/__init__.py +31 -0
  46. pyworkflow/project/config.py +454 -0
  47. pyworkflow/project/manager.py +180 -0
  48. pyworkflow/project/project.py +2095 -0
  49. pyworkflow/project/usage.py +165 -0
  50. pyworkflow/protocol/__init__.py +38 -0
  51. pyworkflow/protocol/bibtex.py +48 -0
  52. pyworkflow/protocol/constants.py +87 -0
  53. pyworkflow/protocol/executor.py +483 -0
  54. pyworkflow/protocol/hosts.py +317 -0
  55. pyworkflow/protocol/launch.py +277 -0
  56. pyworkflow/protocol/package.py +42 -0
  57. pyworkflow/protocol/params.py +781 -0
  58. pyworkflow/protocol/protocol.py +2707 -0
  59. pyworkflow/tests/__init__.py +29 -0
  60. pyworkflow/tests/test_utils.py +25 -0
  61. pyworkflow/tests/tests.py +341 -0
  62. pyworkflow/utils/__init__.py +38 -0
  63. pyworkflow/utils/dataset.py +414 -0
  64. pyworkflow/utils/echo.py +104 -0
  65. pyworkflow/utils/graph.py +169 -0
  66. pyworkflow/utils/log.py +293 -0
  67. pyworkflow/utils/path.py +528 -0
  68. pyworkflow/utils/process.py +153 -0
  69. pyworkflow/utils/profiler.py +92 -0
  70. pyworkflow/utils/progressbar.py +154 -0
  71. pyworkflow/utils/properties.py +617 -0
  72. pyworkflow/utils/reflection.py +129 -0
  73. pyworkflow/utils/utils.py +880 -0
  74. pyworkflow/utils/which.py +229 -0
  75. pyworkflow/webservices/__init__.py +8 -0
  76. pyworkflow/webservices/config.py +8 -0
  77. pyworkflow/webservices/notifier.py +152 -0
  78. pyworkflow/webservices/repository.py +59 -0
  79. pyworkflow/webservices/workflowhub.py +74 -0
  80. pyworkflowtests/tests/__init__.py +0 -0
  81. pyworkflowtests/tests/test_canvas.py +72 -0
  82. pyworkflowtests/tests/test_domain.py +45 -0
  83. pyworkflowtests/tests/test_logs.py +74 -0
  84. pyworkflowtests/tests/test_mappers.py +392 -0
  85. pyworkflowtests/tests/test_object.py +507 -0
  86. pyworkflowtests/tests/test_project.py +42 -0
  87. pyworkflowtests/tests/test_protocol_execution.py +146 -0
  88. pyworkflowtests/tests/test_protocol_export.py +78 -0
  89. pyworkflowtests/tests/test_protocol_output.py +158 -0
  90. pyworkflowtests/tests/test_streaming.py +47 -0
  91. pyworkflowtests/tests/test_utils.py +210 -0
  92. {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/METADATA +2 -2
  93. scipion_pyworkflow-3.11.1.dist-info/RECORD +161 -0
  94. scipion_pyworkflow-3.11.0.dist-info/RECORD +0 -71
  95. {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/WHEEL +0 -0
  96. {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/entry_points.txt +0 -0
  97. {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/licenses/LICENSE.txt +0 -0
  98. {scipion_pyworkflow-3.11.0.dist-info → scipion_pyworkflow-3.11.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,29 @@
1
+ # **************************************************************************
2
+ # *
3
+ # * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
4
+ # *
5
+ # * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
6
+ # *
7
+ # * This program is free software; you can redistribute it and/or modify
8
+ # * it under the terms of the GNU General Public License as published by
9
+ # * the Free Software Foundation; either version 3 of the License, or
10
+ # * (at your option) any later version.
11
+ # *
12
+ # * This program is distributed in the hope that it will be useful,
13
+ # * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ # * GNU General Public License for more details.
16
+ # *
17
+ # * You should have received a copy of the GNU General Public License
18
+ # * along with this program; if not, write to the Free Software
19
+ # * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
20
+ # * 02111-1307 USA
21
+ # *
22
+ # * All comments concerning this program package may be sent to the
23
+ # * e-mail address 'scipion@cnb.csic.es'
24
+ # *
25
+ # **************************************************************************
26
+ """
27
+ Configuration sub-package
28
+ """
29
+
@@ -0,0 +1,37 @@
1
+ #!/usr/bin/env python
2
+
3
+ # **************************************************************************
4
+ # *
5
+ # * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
6
+ # *
7
+ # * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
8
+ # *
9
+ # * This program is free software; you can redistribute it and/or modify
10
+ # * it under the terms of the GNU General Public License as published by
11
+ # * the Free Software Foundation; either version 3 of the License, or
12
+ # * (at your option) any later version.
13
+ # *
14
+ # * This program is distributed in the hope that it will be useful,
15
+ # * but WITHOUT ANY WARRANTY; without even the implied warranty of
16
+ # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17
+ # * GNU General Public License for more details.
18
+ # *
19
+ # * You should have received a copy of the GNU General Public License
20
+ # * along with this program; if not, write to the Free Software
21
+ # * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
22
+ # * 02111-1307 USA
23
+ # *
24
+ # * All comments concerning this program package may be sent to the
25
+ # * e-mail address 'scipion@cnb.csic.es'
26
+ # *
27
+ # **************************************************************************
28
+ """
29
+ Launch the windows with all Projects.
30
+ """
31
+
32
+ from pyworkflow.gui.project import ProjectManagerWindow
33
+
34
+
35
+ if __name__ == '__main__':
36
+
37
+ ProjectManagerWindow().show()
@@ -0,0 +1,51 @@
1
+ #!/usr/bin/env python
2
+
3
+
4
+ import argparse
5
+ from pwem.viewers import EmPlotter
6
+ from pyworkflow.gui.plotter import Plotter
7
+
8
+
9
+ def main():
10
+ parser = argparse.ArgumentParser(prog='Scipion Plot')
11
+ parser.add_argument('--file', help='File to visualize', required=True)
12
+ parser.add_argument('--block', help='Block to visualize')
13
+ parser.add_argument('--type', help='Plot type')
14
+ parser.add_argument('--columns', help='Columns to plot')
15
+ parser.add_argument('--xcolumn', help='X Column to plot')
16
+ parser.add_argument('--orderColumn', help='Column to order')
17
+ parser.add_argument('--orderDir', help='Order direction(ASC, DESC)')
18
+ parser.add_argument('--bins', help='If plot type is histogram, number of bins')
19
+ parser.add_argument('--colors', help='Colors to plot columns')
20
+ parser.add_argument('--styles', help='Styles to plot columns')
21
+ parser.add_argument('--markers', help='Markers to plot columns')
22
+ parser.add_argument('--title', help='Plot title', default='')
23
+ parser.add_argument('--ytitle', help='Y axis title', default='')
24
+ parser.add_argument('--xtitle', help='X axis title', default='')
25
+
26
+ args = parser.parse_args()
27
+ plotfile = args.file
28
+ block = args.block if args.block else ''
29
+ type = args.type
30
+ columns = args.columns
31
+ xcolumn = args.xcolumn
32
+ orderColumn = args.orderColumn
33
+ orderDir = args.orderDir
34
+
35
+ bins = args.bins
36
+ colors = args.colors
37
+ styles = args.styles
38
+ markers = args.markers
39
+ title = args.title
40
+ xtitle = args.xtitle
41
+ ytitle = args.ytitle
42
+
43
+ Plotter.setBackend('TkAgg')
44
+ plotter = EmPlotter.createFromFile(
45
+ plotfile, block, type, columns, colors, styles, markers, xcolumn,
46
+ ytitle, xtitle, title, bins, orderColumn, orderDir)
47
+ plotter.show(block=True)
48
+
49
+
50
+ if __name__ == '__main__':
51
+ main()
@@ -0,0 +1,113 @@
1
+ #!/usr/bin/env python
2
+ # **************************************************************************
3
+ # *
4
+ # * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
5
+ # *
6
+ # * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
7
+ # *
8
+ # * This program is free software; you can redistribute it and/or modify
9
+ # * it under the terms of the GNU General Public License as published by
10
+ # * the Free Software Foundation; either version 3 of the License, or
11
+ # * (at your option) any later version.
12
+ # *
13
+ # * This program is distributed in the hope that it will be useful,
14
+ # * but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
+ # * GNU General Public License for more details.
17
+ # *
18
+ # * You should have received a copy of the GNU General Public License
19
+ # * along with this program; if not, write to the Free Software
20
+ # * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
21
+ # * 02111-1307 USA
22
+ # *
23
+ # * All comments concerning this program package may be sent to the
24
+ # * e-mail address 'scipion@cnb.csic.es'
25
+ # *
26
+ # **************************************************************************
27
+ """
28
+ Launch main project window
29
+ """
30
+
31
+ import sys
32
+ import os
33
+
34
+ from pyworkflow import Config
35
+ from pyworkflow.project import Manager
36
+ from pyworkflow.gui.project import ProjectWindow
37
+ import pyworkflow.utils as pwutils
38
+
39
+ HERE = 'here'
40
+ LAST = 'last'
41
+ LIST = 'list'
42
+
43
+ def openProject(projectName):
44
+ """ Opens a scipion project:
45
+
46
+ :param projectName: Name of an existing project to open,
47
+ or "here" to create a project in the current working dir,
48
+ or "last" to open the most recent project
49
+
50
+ """
51
+ manager = Manager()
52
+ projName = os.path.basename(projectName)
53
+
54
+
55
+ if projName == LIST:
56
+ showProjectList(manager)
57
+ return
58
+ # Handle special name 'here' to create a project
59
+ # from the current directory
60
+ elif projName == HERE:
61
+ cwd = Config.SCIPION_CWD
62
+
63
+ if " " in cwd:
64
+ print("Projects can't have spaces in the name: %s" % cwd)
65
+ sys.exit(1)
66
+
67
+ print("\nYou are trying to create a project here:",
68
+ pwutils.cyan(cwd))
69
+
70
+ if os.listdir(cwd):
71
+ print(pwutils.red('\nWARNING: this folder is not empty!!!'))
72
+ key = input("\nDo you want to create a project here? [y/N]?")
73
+
74
+ if key.lower().strip() != 'y':
75
+ print("\nAborting...")
76
+ sys.exit(0)
77
+ else:
78
+ print("\nCreating project....")
79
+ projName = os.path.basename(cwd)
80
+ projDir = os.path.dirname(cwd)
81
+ manager.createProject(projName, location=projDir)
82
+
83
+ elif projName == LAST: # Get last project
84
+ projects = manager.listProjects()
85
+ if not projects:
86
+ sys.exit("No projects yet, cannot open the last one.")
87
+ projName = projects[0].projName
88
+
89
+ projPath = manager.getProjectPath(projName)
90
+
91
+ if os.path.exists(projPath):
92
+ projWindow = ProjectWindow(projPath)
93
+ projWindow.show()
94
+ else:
95
+ print("Can't open project %s. It does not exist" % projPath)
96
+
97
+ #Show the list of projects
98
+ showProjectList(manager)
99
+
100
+ def showProjectList(manager):
101
+
102
+ projects = manager.listProjects()
103
+
104
+ print("\n******** LIST OF PROJECTS *******\n")
105
+ for project in projects:
106
+ print(project.projName)
107
+ print("\n")
108
+ if __name__ == '__main__':
109
+
110
+ if len(sys.argv) > 1:
111
+ openProject(sys.argv[1])
112
+ else:
113
+ print("usage: pw_project.py PROJECT_NAME or %s or %s or %s" % (HERE, LAST, LIST))
@@ -0,0 +1,143 @@
1
+ #!/usr/bin/env python
2
+ # **************************************************************************
3
+ # *
4
+ # * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
5
+ # *
6
+ # * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
7
+ # *
8
+ # * This program is free software; you can redistribute it and/or modify
9
+ # * it under the terms of the GNU General Public License as published by
10
+ # * the Free Software Foundation; either version 3 of the License, or
11
+ # * (at your option) any later version.
12
+ # *
13
+ # * This program is distributed in the hope that it will be useful,
14
+ # * but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
+ # * GNU General Public License for more details.
17
+ # *
18
+ # * You should have received a copy of the GNU General Public License
19
+ # * along with this program; if not, write to the Free Software
20
+ # * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
21
+ # * 02111-1307 USA
22
+ # *
23
+ # * All comments concerning this program package may be sent to the
24
+ # * e-mail address 'scipion@cnb.csic.es'
25
+ # *
26
+ # **************************************************************************
27
+ ###
28
+ """
29
+ List all existing protocols within Scipion
30
+ """
31
+
32
+ import sys
33
+
34
+ from pwem.protocols import (ProtImport, ProtMicrographs, ProtParticles, Prot2D,
35
+ Prot3D)
36
+ from pyworkflow import Config
37
+ from pyworkflow.viewer import Viewer
38
+ from pyworkflow.protocol.protocol import Protocol
39
+
40
+
41
+ def getFirstLine(doc):
42
+ """ Get the first non empty line from doc. """
43
+ if doc:
44
+ for lines in doc.split('\n'):
45
+ l = lines.strip()
46
+ if l:
47
+ return l
48
+ return ''
49
+
50
+
51
+ def hasDoubleInheritance(classRef):
52
+ # loop while class has single parent
53
+ numParents = len(classRef.__bases__)
54
+ while numParents == 1 and classRef is not Protocol:
55
+ classRef = classRef.__bases__[0]
56
+ numParents = len(classRef.__bases__)
57
+ if numParents > 1:
58
+ return True
59
+ else:
60
+ return False
61
+
62
+
63
+ if __name__ == '__main__':
64
+ count = 0
65
+ withDoc = '--with-doc' in sys.argv
66
+ extended = '--extended' in sys.argv
67
+ ai = '--ai' in sys.argv
68
+
69
+ emProtocolsDict = Config.getDomain().getProtocols()
70
+
71
+ protDict = {}
72
+
73
+ # Group protocols by package name
74
+ for k, v in emProtocolsDict.items():
75
+ packageName = v.getClassPackageName()
76
+
77
+ if packageName not in protDict:
78
+ protDict[packageName] = []
79
+
80
+ if not issubclass(v, Viewer) and not v.isBase():
81
+ if extended:
82
+ protTuple = (k, v, hasDoubleInheritance(v),
83
+ v().allowMpi, v().numberOfMpi,
84
+ v().allowThreads, v().numberOfThreads,
85
+ v().stepsExecutionMode)
86
+ else:
87
+ protTuple = (k, v)
88
+ protDict[packageName].append(protTuple)
89
+
90
+ def iterGroups(protDict):
91
+ groups = list(protDict.keys())
92
+ groups.sort(key=lambda x: 1000-len(protDict[x]))
93
+
94
+ for g in groups:
95
+ yield g, protDict[g]
96
+
97
+ def printProtocols(prots):
98
+ protList = [(p[0], p[1], p[1].getClassLabel()) for p in prots]
99
+ protList.sort(key=lambda x: x[2])
100
+
101
+ for k, v, l in protList:
102
+ doc = getFirstLine(v.__doc__) if withDoc else ''
103
+ print("* link:%s[%s]: %s" % (k, l, doc))
104
+
105
+
106
+ if withDoc:
107
+ for group, prots in iterGroups(protDict):
108
+ print("Package: ", group, "(%d protocols)" % len(prots))
109
+ for p in prots:
110
+ print(" %s ( %s ):" % (p[1].getClassLabel(), p[0]))
111
+ print(" ", p[1].__doc__)
112
+ print("-" * 100)
113
+
114
+ else:
115
+ if extended:
116
+ formatStr = "{:<15}\t{:<35}\t{:<35}" + "\t{:<20}" * 6
117
+ print(formatStr.format("PACKAGE", "PROTOCOL",
118
+ "LABEL", "DOUBLE_INHERITANCE",
119
+ "ALLOWS_MPI", "NUM_MPI",
120
+ "ALLOWS_THREADS", "NUM_THREADS",
121
+ "STEPS_EXEC_MODE"))
122
+ for group, prots in iterGroups(protDict):
123
+ for p in prots:
124
+ print(formatStr.format(group, p[0],
125
+ p[1].getClassLabel(), *p[2:]))
126
+ elif ai:
127
+ for group, prots in iterGroups(protDict):
128
+ for protClassName, protClass in prots:
129
+ print("\nThe protocol named \"%s\" can be found in the %s plugin." %(protClass._label, group))
130
+ print("\"%s\" protocol help is as follows:\n %s." % (protClass._label, protClass.__doc__))
131
+ instance = protClass()
132
+
133
+ for name, param in instance._definition.iterParams():
134
+ print("\"%s\" has a \"%s\" parameter that is explained as: %s" % (protClass._label, param.label, param.help))
135
+
136
+
137
+
138
+ else:
139
+ formatStr = "{:<15}\t{:<35}\t{:<35}"
140
+ print(formatStr.format("PACKAGE", "PROTOCOL", "LABEL"))
141
+ for group, prots in iterGroups(protDict):
142
+ for k, v in prots:
143
+ print(formatStr.format(group, k, v.getClassLabel()))
@@ -0,0 +1,51 @@
1
+ #!/usr/bin/env python
2
+ # **************************************************************************
3
+ # *
4
+ # * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
5
+ # *
6
+ # * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
7
+ # *
8
+ # * This program is free software; you can redistribute it and/or modify
9
+ # * it under the terms of the GNU General Public License as published by
10
+ # * the Free Software Foundation; either version 3 of the License, or
11
+ # * (at your option) any later version.
12
+ # *
13
+ # * This program is distributed in the hope that it will be useful,
14
+ # * but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
+ # * GNU General Public License for more details.
17
+ # *
18
+ # * You should have received a copy of the GNU General Public License
19
+ # * along with this program; if not, write to the Free Software
20
+ # * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
21
+ # * 02111-1307 USA
22
+ # *
23
+ # * All comments concerning this program package may be sent to the
24
+ # * e-mail address 'scipion@cnb.csic.es'
25
+ # *
26
+ # **************************************************************************
27
+ """
28
+ This module is responsible for launching protocol executions.
29
+ """
30
+ import sys
31
+ from pyworkflow.utils import LoggingConfigurator
32
+
33
+ if __name__ == '__main__':
34
+
35
+ if len(sys.argv) == 6:
36
+ projPath = sys.argv[1]
37
+ dbPath = sys.argv[2]
38
+ protId = int(sys.argv[3])
39
+ stdOut = sys.argv[4]
40
+ stdErr = sys.argv[5]
41
+
42
+ LoggingConfigurator.setUpProtocolRunLogging(stdOut, stdErr)
43
+
44
+ from pyworkflow.protocol import runProtocolMain
45
+ runProtocolMain(projPath, dbPath, protId)
46
+
47
+ else:
48
+ from os.path import basename
49
+ print("usage: %s projPath dbPath protocolID pathToStdoutLog pathToStdErrLog." % basename(sys.argv[0]))
50
+ print("sys.argv: %s" % sys.argv)
51
+ sys.exit(1)
@@ -0,0 +1,267 @@
1
+ #!/usr/bin/env python
2
+ # **************************************************************************
3
+ # *
4
+ # * Authors: J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
5
+ # *
6
+ # * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
7
+ # *
8
+ # * This program is free software; you can redistribute it and/or modify
9
+ # * it under the terms of the GNU General Public License as published by
10
+ # * the Free Software Foundation; either version 3 of the License, or
11
+ # * (at your option) any later version.
12
+ # *
13
+ # * This program is distributed in the hope that it will be useful,
14
+ # * but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
+ # * GNU General Public License for more details.
17
+ # *
18
+ # * You should have received a copy of the GNU General Public License
19
+ # * along with this program; if not, write to the Free Software
20
+ # * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
21
+ # * 02111-1307 USA
22
+ # *
23
+ # * All comments concerning this program package may be sent to the
24
+ # * e-mail address 'scipion@cnb.csic.es'
25
+ # *
26
+ # **************************************************************************
27
+
28
+ """
29
+ Run or show the selected tests. Tests can be selected by giving
30
+ the "case", or by giving the paths and file pattern to use for
31
+ searching them.
32
+ """
33
+ from pyworkflow.utils import LoggingConfigurator
34
+ import argparse
35
+ from collections import OrderedDict
36
+
37
+ import pyworkflow.tests as pwtests
38
+ from pyworkflow import getTestsScript, SCIPION_TESTS_CMD, Config
39
+
40
+ from pyworkflow.tests import *
41
+
42
+ MODULE = 0
43
+ CLASS = 1
44
+ TEST = 2
45
+
46
+
47
+ class Tester:
48
+ def main(self, args=None):
49
+
50
+ # Trigger plugin's variable definition
51
+ Config.getDomain().getPlugins()
52
+
53
+ parser = argparse.ArgumentParser(prog=self.getTestsCommand(), description=__doc__)
54
+ g = parser.add_mutually_exclusive_group()
55
+ g.add_argument('--run', action='store_true', help='run the selected tests')
56
+ g.add_argument('--show', action='store_true', help='show available tests',
57
+ default=True)
58
+
59
+ add = parser.add_argument # shortcut
60
+
61
+ add('--pattern', default='test*.py',
62
+ help='pattern for the files that will be used in the tests')
63
+ add('--grep', default=None, nargs='+',
64
+ help='only show/run tests containing the provided words')
65
+ add('--skip', default=None, nargs='+',
66
+ help='skip tests that contains these words')
67
+ add('--log', default=None, nargs='?',
68
+ help="Generate logs files with the output of each test.")
69
+ add('--mode', default='classes', choices=['modules', 'classes', 'onlyclasses', 'all'],
70
+ help='how much detail to give in show mode')
71
+ add('tests', metavar='TEST', nargs='*',
72
+ help='test case from string identifier (module, class or callable)')
73
+ args = parser.parse_args(args)
74
+
75
+ if not args.run and not args.show and not args.tests:
76
+ sys.exit(parser.format_help())
77
+
78
+ # Logging stuff first
79
+ self.log = args.log
80
+
81
+ if self.log:
82
+ LoggingConfigurator.setupLogging(logFile=self.log)
83
+ else:
84
+ logging.basicConfig(level=Config.SCIPION_LOG_LEVEL, format=Config.SCIPION_LOG_FORMAT)
85
+
86
+ self.logger = logging.getLogger(__name__)
87
+
88
+ # This goes intentionally to the output. Is not a logging line._S
89
+ print("Scanning tests...")
90
+
91
+
92
+ testsDict = OrderedDict()
93
+ testLoader = unittest.defaultTestLoader
94
+
95
+ if args.tests:
96
+ print(pwutils.cyanStr("Loading test/s %s" % args.tests))
97
+
98
+ # In this case the tests are passed as argument.
99
+ # The full name of the test will be used to load it.
100
+ testsDict['tests'] = unittest.TestSuite()
101
+ for t in args.tests:
102
+ try:
103
+ testsDict['tests'].addTests(testLoader.loadTestsFromName(t))
104
+ except Exception as e:
105
+ self.logger.error('Cannot load test %s -- skipping' % t, exc_info=True)
106
+ else:
107
+ # In this other case, we will load the test available
108
+ # from pyworkflow and the other plugins
109
+ # self.paths = [('pyworkflow', os.path.dirname(os.path.dirname(pw.__file__)))]
110
+ self.paths = []
111
+ for name, plugin in pw.Config.getDomain().getPlugins().items():
112
+ self.paths.append((name, os.path.dirname(plugin.__path__[0])))
113
+ for k, p in self.paths:
114
+ testPath = os.path.join(p, k, 'tests')
115
+ if os.path.exists(testPath):
116
+ self.logger.debug("Discovering tests at %s" % testPath)
117
+ testsDict[k] = testLoader.discover(testPath,
118
+ pattern=args.pattern,
119
+ top_level_dir=p)
120
+
121
+ self.grep = [g.lower() for g in args.grep] if args.grep else []
122
+ self.skip = [s.lower() for s in args.skip] if args.skip else []
123
+ self.mode = args.mode
124
+
125
+ if args.tests:
126
+ self.runSingleTest(testsDict['tests'])
127
+
128
+ elif args.run:
129
+ for moduleName, tests in testsDict.items():
130
+ self.logger.info(pwutils.cyanStr(">>>> %s" % moduleName))
131
+ self.runTests(moduleName, tests)
132
+
133
+ elif args.grep:
134
+ pattern = args.grep[0]
135
+ for moduleName, tests in testsDict.items():
136
+ self.printTests(pattern, tests)
137
+
138
+ else:
139
+ for moduleName, tests in testsDict.items():
140
+ if self._match(moduleName):
141
+ print(pwutils.cyanStr(">>>> %s" % moduleName))
142
+ self.printTests(moduleName, tests)
143
+
144
+ def _match(self, itemName):
145
+ # Add a space to allow " tomo." as a filter for example to narrow search to tomo and leaving out xmipptomo, emntomo, ...
146
+ itemLower = " " + itemName.lower()
147
+ grep = (not self.grep or
148
+ all(g in itemLower for g in self.grep))
149
+ skip = (self.skip and
150
+ any(s in itemLower for s in self.skip))
151
+
152
+ return grep and not skip
153
+
154
+ def __iterTests(self, test):
155
+ """ Recursively iterate over a testsuite. """
156
+ self.logger.debug("__iterTests: %s, is-suite: %s" % (str(test.__class__),
157
+ isinstance(test, unittest.TestSuite)))
158
+
159
+ if isinstance(test, unittest.TestSuite):
160
+ for t in test:
161
+ self.__iterTests(t)
162
+ else:
163
+ yield test
164
+
165
+ def _visitTests(self, moduleName, tests, newItemCallback):
166
+ """ Show the list of tests available """
167
+ mode = self.mode
168
+
169
+ assert mode in ['modules', 'classes', 'onlyclasses', 'all'], 'Unknown mode %s' % mode
170
+
171
+ # First flatten the list of tests.
172
+ # testsFlat = list(iter(self.__iterTests(tests)))
173
+
174
+ testsFlat = []
175
+ toCheck = [t for t in tests]
176
+
177
+ while toCheck:
178
+ test = toCheck.pop()
179
+ if isinstance(test, unittest.TestSuite):
180
+ toCheck += [t for t in test]
181
+ elif test not in testsFlat:
182
+ testsFlat.append(test)
183
+
184
+ # Follow the flattened list of tests and show the module, class
185
+ # and name, in a nice way.
186
+ lastClass = None
187
+ lastModule = None
188
+ if testsFlat:
189
+ for t in testsFlat:
190
+
191
+ testModuleName, className, testName = t.id().rsplit('.', 2)
192
+
193
+ # If there is a failure loading the test, show it
194
+ errorStr = 'unittest.loader._FailedTest.'
195
+ if testModuleName.startswith(errorStr):
196
+ newName = t.id().replace(errorStr, '')
197
+ if self._match(newName):
198
+ self.logger.error(
199
+ pwutils.redStr('Error loading the test. Please, run the test for more information: ' + newName))
200
+ continue
201
+
202
+ if testModuleName != lastModule:
203
+ lastModule = testModuleName
204
+ if mode != 'onlyclasses':
205
+ newItemCallback(MODULE, "%s" % testModuleName)
206
+
207
+ if mode in ['classes', 'onlyclasses', 'all'] and className != lastClass:
208
+ lastClass = className
209
+ newItemCallback(CLASS, "%s.%s"
210
+ % (testModuleName, className))
211
+
212
+ if mode == 'all':
213
+ newItemCallback(TEST, "%s.%s.%s"
214
+ % (testModuleName, className, testName))
215
+ else:
216
+ if not self.grep:
217
+ self.logger.warning(pwutils.greenStr(' The plugin does not have any test'))
218
+
219
+ def _printNewItem(self, itemType, itemName):
220
+ if self._match(itemName):
221
+ spaces = (itemType * 2) * ' '
222
+ # Do not use intentionally the logger. This is not a logging output but a GUI output
223
+ print("%s %s %s" % (spaces, self.getTestsCommand(), itemName))
224
+
225
+ def getTestsCommand(self):
226
+ return os.environ.get(SCIPION_TESTS_CMD, getTestsScript())
227
+
228
+ def printTests(self, moduleName, tests):
229
+ self._visitTests(moduleName, tests, self._printNewItem)
230
+
231
+ def _runNewItem(self, itemType, itemName):
232
+ if self._match(itemName):
233
+ spaces = (itemType * 2) * ' '
234
+ script = getTestsScript()
235
+ cmd = "%s %s %s %s" % (pw.PYTHON, script, spaces, itemName)
236
+ run = ((itemType == MODULE and self.mode == 'modules') or
237
+ (itemType == CLASS and self.mode in ('classes', 'onlyclasses')) or
238
+ (itemType == TEST and self.mode == 'all'))
239
+ if run:
240
+ if self.log:
241
+ # logFile = join(self.testsDir, '%s.txt' % itemName)
242
+ cmdFull = cmd + " >> %s 2>&1" % self.log
243
+ else:
244
+ logFile = ''
245
+ cmdFull = cmd
246
+
247
+ self.logger.info(pwutils.greenStr(cmdFull))
248
+ t = pwutils.Timer()
249
+ t.tic()
250
+ self.testCount += 1
251
+ os.system(cmdFull)
252
+
253
+ def runTests(self, moduleName, tests):
254
+
255
+ self.testCount = 0
256
+ self._visitTests(moduleName, tests, self._runNewItem)
257
+
258
+ def runSingleTest(self, tests):
259
+ result = pwtests.GTestResult()
260
+ tests.run(result)
261
+ result.doReport()
262
+ resultPassed = result.numberTests - result.testFailed
263
+ sys.exit(1 if result.testFailed > 0 or resultPassed == 0 else 0)
264
+
265
+
266
+ if __name__ == '__main__':
267
+ Tester().main()