scipion-pyworkflow 3.7.0__py3-none-any.whl

pyworkflowtests/tests/test_protocol_output.py

@@ -0,0 +1,158 @@
+# **************************************************************************
+# *
+# * Authors:     Pablo Conesa (pconesa@cnb.csic.es) [1]
+# *              J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [2]
+# *
+# * [1] Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# * [2] SciLifeLab, Stockholm University
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+import os
+
+import pyworkflow as pw
+import pyworkflow.object as pwobj
+import pyworkflow.tests as pwtests
+import pyworkflow.mapper as pwmapper
+import pyworkflow.protocol as pwprot
+from pyworkflow.project import Project
+from pyworkflowtests.protocols import ProtOutputTest
+from pyworkflowtests import Domain, MockObject
+
+
+# Protocol to output of basic scipion objects
+class TestProtocolOutputs(pwtests.BaseTest):
+    @classmethod
+    def setUpClass(cls):
+        pwtests.setupTestProject(cls, writeLocalConfig=True)
+        pwtests.setupTestOutput(cls)
+
+        # Set the application domain
+        pw.Config.setDomain("pyworkflowtests")
+
+    def test_basicObjectOutput(self):
+        """Test the list with several Complex"""
+        fn = self.getOutputPath("protocol.sqlite")
+
+        # Discover objects and protocols
+        mapperDict = Domain.getMapperDict()
+
+        mapper = pwmapper.SqliteMapper(fn, mapperDict)
+        # Associate the project
+        proj = Project(Domain, path=self.getOutputPath(''))
+
+        prot = ProtOutputTest(mapper=mapper, n=2, project=proj,
+                              workingDir=self.getOutputPath(''))
+
+        # Add and old style o, not in the outputs dictionary
+        prot.output1 = MockObject()
+
+        self.assertFalse(prot._useOutputList.get(),
+                         "useOutputList wrongly initialized")
+
+        outputs = [o for o in prot.iterOutputAttributes()]
+        self.assertTrue(1, len(outputs))
+
+        prot._stepsExecutor = pwprot.StepExecutor(hostConfig=None)
+
+        #Create the logs folder
+        prot.makeWorkingDir()
+
+        prot.run()
+
+        self.assertEqual(prot._steps[0].getStatus(),
+                         pwprot.STATUS_FINISHED)
+
+        # Check there is an output
+        self.assertOutput(prot)
+
+        outputs = [o for o in prot.iterOutputAttributes()]
+
+        # We are intentionally ignoring a protocol with o (EMObject)
+        # That has been continued, We do not find a real case now.
+        self.assertEqual(1, len(outputs),
+                         msg="Integer o not registered properly.")
+
+        outputs = [o for o in prot.iterOutputAttributes(pwobj.Integer)]
+
+        # Test passing a filter
+        self.assertEqual(1, len(outputs),
+                         msg="Integer not matched when filtering outputs.")
+
+        # Test with non existing class
+        class NotRealClass:
+            pass
+
+        outputs = [o for o in prot.iterOutputAttributes(NotRealClass)]
+
+        # Test passing a class
+        self.assertEqual(0, len(outputs),
+                         msg="Filter by class in iterOutputAttributes does "
+                             "not work.")
+
+        self.assertTrue(prot._useOutputList.get(),
+                        "useOutputList not activated")
+
+    def test_basicObjectInProject(self):
+        prot = self.newProtocol(ProtOutputTest,
+                                objLabel='to generate basic input')
+        print("working dir: %s" % prot.getWorkingDir())
+        # Define a negative output for later tests
+        prot._defineOutputs(negative=pwobj.Integer(-20))
+        self.launchProtocol(prot)
+
+        # Default value is 10 so output is 20
+        self.assertOutput(prot)
+
+        # Second protocol to test linking
+        prot2 = self.newProtocol(ProtOutputTest,
+                                 objLabel='to read basic input')
+
+        # Set the pointer for the integer
+        prot2.iBoxSize.setPointer(pwobj.Pointer(prot, extended="oBoxSize"))
+        self.launchProtocol(prot2)
+        self.assertOutput(prot2, value=40)
+
+        # Test validation: only positive numbers are allowed
+        prot3 = self.newProtocol(ProtOutputTest,
+                                 objLabel='invalid input',
+                                 iBoxSize=-10)
+        # We expect this to fail
+        with self.assertRaises(Exception):
+            self.launchProtocol(prot3)
+        # Test validation: pointer value is validated
+        prot4 = self.newProtocol(ProtOutputTest,
+                                 objLabel='invalid pointer input')
+        # Now use negative pointer output
+        prot4.iBoxSize.setPointer(pwobj.Pointer(prot, extended="negative"))
+
+        # We expect this to fail
+        with self.assertRaises(Exception):
+            self.launchProtocol(prot4)
+
+    def assertOutput(self, prot, value=20):
+        # Check there is an output
+        self.assertTrue(hasattr(prot, 'oBoxSize'),
+                        msg="Protocol output boxSize (OInteger) not registered"
+                            " as attribute.")
+
+        self.assertEqual(value, prot.oBoxSize.get(),
+                         "oBoxSize value is wrong: %s , expected %s" %
+                         (prot.oBoxSize, value))

pyworkflowtests/tests/test_streaming.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
+# *
+# * [1] SciLifeLab, Stockholm University
+# *
+# * This program is free software: you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation, either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+import pyworkflow.tests as pwtests
+
+from pyworkflow import Config
+from pyworkflowtests.protocols import  ConcurrencyProtocol
+
+
+class TestConcurrency(pwtests.BaseTest):
+    
+    @classmethod
+    def setUpClass(cls):
+        pwtests.setupTestOutput(cls)
+
+    
+       # Set the application domain
+        Config.setDomain("pyworkflowtests")
+        pwtests.setupTestProject(cls)
+
+
+    def test_simple_steps_concurrency(self):
+        prot = self.newProtocol(ConcurrencyProtocol, numberOfThreads=3)
+
+        self.launchProtocol(prot)

pyworkflowtests/tests/test_utils.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# coding: latin-1
+"""
+Created on Mar 25, 2014
+
+@author: airen
+@author: roberto.marabini
+"""
+import datetime
+import os
+from subprocess import Popen
+from io import StringIO
+
+from pyworkflow import APPS, Variable
+from pyworkflow.utils.process import killWithChilds
+from pyworkflow.tests import *
+from pyworkflow.utils import utils, prettyDict, getListFromValues, strToDuration
+from pyworkflow.utils import ProgressBar
+
+
+class TestBibtex(BaseTest):
+    """ Some minor tests to the bibtexparser library. """
+
+    @classmethod
+    def setUpClass(cls):
+        setupTestOutput(cls)
+        
+    def test_Parsing(self):
+        bibtex = """
+
+@article{delaRosaTrevin2013,
+title = "Xmipp 3.0: An improved software suite for image processing in electron microscopy ",
+journal = "Journal of Structural Biology ",
+volume = "184",
+number = "2",
+pages = "321 - 328",
+year = "2013",
+issn = "1047-8477",
+doi = "http://dx.doi.org/10.1016/j.jsb.2013.09.015",
+url = "http://www.sciencedirect.com/science/article/pii/S1047847713002566",
+author = "J.M. de la Rosa-Trevín and J. Otón and R. Marabini and A. Zaldívar and J. Vargas and J.M. Carazo and C.O.S. Sorzano",
+keywords = "Electron microscopy, Single particles analysis, Image processing, Software package "
+}
+
+@incollection{Sorzano2013,
+title = "Semiautomatic, High-Throughput, High-Resolution Protocol for Three-Dimensional Reconstruction of Single Particles in Electron Microscopy",
+booktitle = "Nanoimaging",
+year = "2013",
+isbn = "978-1-62703-136-3",
+volume = "950",
+series = "Methods in Molecular Biology",
+editor = "Sousa, Alioscka A. and Kruhlak, Michael J.",
+doi = "10.1007/978-1-62703-137-0_11",
+url = "http://dx.doi.org/10.1007/978-1-62703-137-0_11",
+publisher = "Humana Press",
+keywords = "Single particle analysis; Electron microscopy; Image processing; 3D reconstruction; Workflows",
+author = "Sorzano, CarlosOscar and Rosa Trevín, J.M. and Otón, J. and Vega, J.J. and Cuenca, J. and Zaldívar-Peraza, A. and Gómez-Blanco, J. and Vargas, J. and Quintana, A. and Marabini, Roberto and Carazo, JoséMaría",
+pages = "171-193",
+}
+"""
+
+        prettyDict(utils.parseBibTex(bibtex))
+
+
+class TestProccess(BaseTest):
+    """ Some tests for utils.process module. """
+
+    @classmethod
+    def setUpClass(cls):
+        setupTestOutput(cls)
+        
+    def test_Process(self):
+        prog = pw.join(APPS, 'pw_sleep.py')
+        p = Popen('python %s 500' % prog, shell=True)
+        print("pid: %s" % p.pid)
+        time.sleep(5)
+        killWithChilds(p.pid)
+
+
+class TestGetListFromRangeString(BaseTest):
+
+    def test_getListFromRangeString(self):
+        inputStrings = ["1,5-8,10"        , "2,6,9-11"       , "2 5, 6-8"     , "1-4 8"]
+        outputLists = [[1, 5, 6, 7, 8, 10], [2, 6, 9, 10, 11], [2, 5, 6, 7, 8], [1,2,3,4, 8]]
+
+        for s, o in zip(inputStrings, outputLists):
+            self.assertEqual(o, pwutils.getListFromRangeString(s))
+            # Check that also works properly with spaces as delimiters
+            s2 = s.replace(',', ' ')
+            self.assertEqual(o, pwutils.getListFromRangeString(s2))
+
+
+class TestListFromValues(unittest.TestCase):
+    """ Tests list created from str"""
+
+    def _callAndAssert(self, strValue, expected, length=None, caster=str):
+
+        result = getListFromValues( strValue, length, caster)
+
+        self.assertEqual(result, expected, "List from string does not work for %s" % strValue)
+
+    def test_getListFromValues(self):
+        """ Test numeric list definitions like:
+            '1 1 2x2 4 4' -> ['1', '1', '2', '2', '4', '4']
+            '2x3, 3x4, 1' -> ['3', '3', '4', '4', '4', '1']"
+        """
+
+        self._callAndAssert('1 1 2x2 4 4', ['1', '1', '2x2', '4', '4'])
+        self._callAndAssert('1 1 2x2 4 4', [1, 1, 2, 2, 4, 4], caster=int)
+        self._callAndAssert('2,3,4,1', [2, 3, 4, 1], caster=int)
+        self._callAndAssert('2 , 3 , 4 , 1', [2, 3, 4, 1], caster=int)
+        self._callAndAssert('2,3.3,4', [2.0, 3.3, 4.0], caster=float)
+
+
+
+class TestProgressBar(unittest.TestCase):
+
+    def caller(self, total, step, fmt, resultGold):
+        ti = time.time()
+        result = StringIO()
+        pb = ProgressBar(total=total, fmt=fmt, output=result,
+                         extraArgs={'objectId': 33})
+
+        pb.start()
+        for i in range(total):
+            if i % step == 0:
+                pb.update(i+1)
+        pb.finish()
+        self.assertEqual(resultGold.strip(), result.getvalue().strip())
+        result.close()
+        tf = time.time()
+        print("%d iterations in %f sec" % (total, tf - ti))
+
+    def test_dot(self):
+        total = 1000000
+        step = 10000
+        ratio = int(total/step)
+        resultGold = '.' * (ratio+1)
+        self.caller(total=total, step=step,
+                    fmt=ProgressBar.DOT, resultGold=resultGold)
+
+    def test_default(self):
+        total = 3
+        step = 1
+        resultGold = ('\rProgress: [                                        ] '
+                      '  0%\rProgress: [=============                         '
+                      '  ]  33%\rProgress: [==========================        '
+                      '      ]  66%\rProgress: [=============================='
+                      '==========] 100%')
+        self.caller(total=total, step=step,
+                    fmt=ProgressBar.DEFAULT, resultGold=resultGold)
+
+    def test_full(self):
+        total = 3
+        step = 1
+        resultGold = ('\r[                                        ] 0/3 (  0%)'
+                      ' 3 to go\r[=============                           ] '
+                      '1/3 ( 33%) 2 to go\r[==========================      '
+                      '        ] 2/3 ( 66%) 1 to go\r[======================'
+                      '==================] 3/3 (100%) 0 to go')
+        self.caller(total=total, step=step,
+                    fmt=ProgressBar.FULL, resultGold=resultGold)
+
+    def test_objectid(self):
+        total = 3
+        step = 1
+        ratio = int(total/step)
+        resultGold = ('\r[                                        ] 0/3 (  0%)'
+                      ' (objectId=33)\r[=============                         '
+                      '  ] 1/3 ( 33%) (objectId=33)\r[========================'
+                      '==              ] 2/3 ( 66%) (objectId=33)\r[=========='
+                      '==============================] 3/3 (100%) '
+                      '(objectId=33)')
+        self.caller(total=total, step=step,
+                    fmt=ProgressBar.OBJID, resultGold=resultGold)
+
+
+
+class TestPathTools(unittest.TestCase):
+
+    def test_filemodificationtime(self):
+
+        # Test is file closed
+
+        import tempfile
+        import time
+
+        since = datetime.datetime.now()
+        time.sleep(1)
+
+        tmpFile = tempfile.NamedTemporaryFile()
+        self.assertFalse(pwutils.isFileFinished(tmpFile.name), "File is NOT finished")
+        time.sleep(1)
+
+        self.assertTrue(pwutils.isFileFinished(tmpFile.name, duration=0.5), "File is finished after 2 seconds")
+
+
+        self.assertTrue(pwutils.hasChangedSince(tmpFile.name, since ), "hasChanged should have returned true. False negative.")
+        since = datetime.datetime.now()
+        self.assertFalse(pwutils.hasChangedSince(tmpFile.name, since ), "hasChanged should have returned false. False positive.")
+
+
+
+    def test_durationstrings(self):
+
+        self.assertEqual(70, strToDuration("1m 10s"), "String duration wrongly converted")
+
+
+
+