scipion-pyworkflow 3.7.0__py3-none-any.whl

pyworkflow/webservices/notifier.py

@@ -0,0 +1,162 @@
+# **************************************************************************
+# *
+# * Authors:    Roberto Marabini       (roberto@cnb.csic.es)
+#               J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+
+import os
+import time
+import threading
+import uuid
+from datetime import timedelta, datetime
+from urllib.parse import urlencode
+from urllib.request import build_opener, HTTPHandler
+
+import pyworkflow
+import pyworkflow.utils as pwutils
+from . import config
+
+
+class ProjectWorkflowNotifier(object):
+    """ Implement different types of notifications about a given
+    project. Currently, the protocols in a workflow will be sent.
+    """
+
+    def __init__(self, project):
+        self.project = project
+
+    def _getUuidFileName(self):
+        return self.project.getLogPath("uuid.log")
+
+    def _getDataFileName(self, fileName="data.log"):
+        return self.project.getLogPath(fileName)
+
+    def _getUuid(self):
+        # Load (or create if not exits) a file
+        # in the project Logs folder to store an unique
+        # project identifier
+        uuidFn = self._getUuidFileName()
+        try:
+            with open(uuidFn) as f:
+                uuidValue = f.readline()
+        except IOError:
+            uuidValue = str(uuid.uuid4())
+            with open(uuidFn, 'w') as f:
+                f.write(uuidValue)
+
+        return uuidValue
+
+    def _modifiedBefore(self, seconds):
+        """ Return True if the uuid.log file has been modified within a given
+        number of seconds. """
+        uuidFn = self._getUuidFileName()
+        if not os.path.exists(uuidFn):
+            return False
+        mTime = datetime.fromtimestamp(os.path.getmtime(uuidFn))
+        delta = datetime.now() - mTime
+
+        return delta < timedelta(seconds=seconds)
+
+    def _sendData(self, url, dataDict=None):
+        try:
+            # then connect to webserver a send json
+            # set debuglevel=0 for no messages
+            opener = build_opener(HTTPHandler(debuglevel=0))
+            data = urlencode(dataDict).encode()
+            content = opener.open(url, data=data).read()
+            now = time.time()
+            os.utime(self._getUuidFileName(), (now, now))
+        except Exception as e:
+            pass
+            # print("Could not notify, maybe there is not internet connection.")
+
+    def _dataModified(self, projectWorfklow):
+        try:
+            with open(self._getDataFileName()) as f:
+                projectWorfklow2 = f.readline()
+                if projectWorfklow2 == projectWorfklow:
+                    return False
+        except IOError:
+            pass
+        return True
+
+    def notifyWorkflow(self):
+
+        try:
+            # check if environment exists otherwise abort
+            if not pyworkflow.Config.SCIPION_NOTIFY:
+                return
+
+            # if project specifies not to send stats
+            if self._isProjectMuted():
+                return
+            # Check the seconds range of the notify, by default one day
+            seconds = int(os.environ.get('SCIPION_NOTIFY_SECONDS', '86400'))
+
+            if self._modifiedBefore(seconds):  # notify not more than once a day
+                return
+
+            # INFO: now we are only sending the protocols names in the project.
+            # We could pass namesOnly=False to get the full workflow template
+            projectWorfklow = self.project.getProtocolsJson(namesOnly=True)
+
+            # if list with workflow has not been altered do not sent it
+            if not self._dataModified(projectWorfklow):
+                return
+            else:
+                # For compatibility with version 1.0 check
+                # if Log directory exists. If it does not
+                # create it
+                # TODO: REMOVE this check in scipion 1.3
+                dataFile = self._getDataFileName()
+                # create the folder of the file path if not exists
+                pwutils.makeFilePath(dataFile)
+                with open(dataFile, 'w') as f:
+                    f.write(projectWorfklow)
+            dataDict = {'project_uuid': self._getUuid(),
+                        'project_workflow': projectWorfklow}
+
+            urlName = os.environ.get('SCIPION_NOTIFY_URL',
+                                     config.SCIPION_STATS_WORKFLOW_APP).strip()
+            urlName += "addOrUpdateWorkflow/"
+            t = threading.Thread(name="notifier", target=lambda: self._sendData(urlName, dataDict))
+            t.start()  # will execute function in a separate thread
+        except Exception as e:
+            print("Can't report usage: ", e)
+
+    def _isProjectMuted(self):
+        """ Projects are muted if they come from tests, Since there is no flag for it
+        we will assume that if the project name starts with Test it will be considered
+        a test and therefore no statistics will be sent"""
+        return os.path.basename(self.project.name).startswith("Test")
+
+    def getEntryFromWebservice(self, uuid):
+        if not pyworkflow.Config.SCIPION_NOTIFY:
+            return
+        urlName = os.environ.get('SCIPION_NOTIFY_URL').strip()
+        # remove last directory
+        urlName = os.path.split(urlName)[0]
+        url = urlName + "/?project_uuid=" + uuid
+        resultDict = self._sendData(url)

pyworkflow/webservices/repository.py

@@ -0,0 +1,59 @@
+import webbrowser
+import json
+import requests
+
+from pyworkflow import CORE_VERSION
+from . import config
+
+
+class WorkflowRepository(object):
+    """ Manager to communicate with the workflow repository services.
+    It will provide functions to:
+    - Search workflows (open the url in a browser).
+    - Upload a given workflow json file.
+    """
+    def __init__(self,
+                 repositoryUrl=config.WORKFLOW_REPOSITORY_SERVER,
+                 uploadFileSuffix=config.WORKFLOW_PROG_STEP1,
+                 uploadMdSuffix=config.WORKFLOW_PROG_STEP2):
+        self._url = repositoryUrl
+        self._uploadFileUrl = repositoryUrl + uploadFileSuffix
+        self._uploadMdUrl = repositoryUrl + uploadMdSuffix
+
+    def search(self):
+        """ Open the repository URL in a web browser. """
+        webbrowser.open(self._url)
+
+    def upload(self, jsonFileName):
+        """ Upload a given workflow providing the path ot the json file.
+
+        First the file is uploaded, then the metadata is uploaded.
+        The script uploads the file and then opens a browser for the metadata
+        Note that the two steps are needed since no initial value can be passed
+        to a file field. poster3 module is needed. Poster3 is pure python
+        so it may be added to the directory rather than installed if needed.
+
+        The server is django a uses filefield and csrf_exempt.
+        csrf_exempt disable csrf checking. filefield
+        """
+
+        # we are going to upload a file so this is a multipart
+        # connection
+        with open(jsonFileName, "rb") as workflowFile:
+            file_dict = {"json": workflowFile}
+            response = requests.post(self._uploadFileUrl, files=file_dict)
+
+        # server returns  a json stored as text at response.text
+        _dict = json.loads(response.text)
+
+        version = CORE_VERSION
+        # version hack end
+
+        fnUrl = "?jsonFileName=%s&versionInit=%s" % (_dict['jsonFileName'],
+                                                     version)  # use GET
+        # open browser to fill metadata, fileName will be saved as session
+        # variable. Note that I cannot save the file never in the
+        # session in the first connection because the browser changes
+        # from urlib2 to an actual browser
+        # so sessions are different
+        webbrowser.open(self._uploadMdUrl + fnUrl)

pyworkflow/webservices/workflowhub.py

@@ -0,0 +1,74 @@
+import json
+from urllib.request import Request, urlopen
+from pyworkflow.template import Template
+
+
+class WHTemplate(Template):
+
+    def __init__(self, source, name, description, url):
+        super().__init__(source, name, description)
+        self.url = url
+
+    def loadContent(self):
+        """ Download the file pointer by url and read the content"""
+
+        return make_request(self.url, asJson=False)
+
+
+def make_request(url, asJson=True):
+    """ Makes a request to the url and returns the json as a dictionary"""
+
+    req = Request(url)
+    req.add_header("accept", "application/json")
+
+    with urlopen(req, timeout=1) as response:
+        if asJson:
+            data = json.load(response)
+        else:
+            html_response = response.read()
+            encoding = response.headers.get_content_charset('utf-8')
+            data = html_response.decode(encoding)
+        return data
+
+
+def get_workflow_file_url(workflow_id, version):
+    root_url = "https://workflowhub.eu/workflows/%s/git/%s/" % (workflow_id, version)
+    url = root_url + 'tree'
+
+    result = make_request(url)
+
+    for file in result["tree"]:
+        path = (file["path"])
+        if path.endswith(".json.template"):
+            return root_url + 'raw/' + path
+
+
+def get_wh_templates(template_id=None, organization="Scipion%20CNB"):
+    """ Returns a list of scipion templates available in workflow hub"""
+
+    url = "https://workflowhub.eu/ga4gh/trs/v2/tools?organization=%s" % organization
+
+    response = make_request(url)
+
+    template_list = []
+
+    for workflow in response:
+        workflow_id = workflow["id"]
+        name = workflow["name"]
+        description = workflow['description']
+        version = workflow["versions"][-1]
+        version_id = version["id"]
+        template_url = get_workflow_file_url(workflow_id, version_id)
+
+        new_template = WHTemplate("Workflow hub", name, description, template_url)
+        if template_id is None or new_template.getObjId() == template_id:
+            template_list.append(new_template)
+
+    return template_list
+
+
+if __name__ == "__main__":
+
+    templates = get_wh_templates()
+    for template in templates:
+        print(template)

pyworkflow/wizard.py

@@ -0,0 +1,64 @@
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *              Jose Gutierrez (jose.gutierrez@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+This module is mainly for the Viewer class, which 
+serve as base for implementing visualization tools(Viewer sub-classes).
+"""
+
+DESKTOP_TKINTER = 'tkinter'
+WEB_DJANGO = 'django'
+
+
+class Wizard(object):
+    """ This is a special case of GUI to help the user selecting important parameters.
+
+    The _targets attribute is used to define to which parameters the Wizard can deal with.
+    It will be a list of tuples such as::
+
+        _targets = [(DefImportMicrographs, ['voltage', sphericalAberration']), (DefCTFMicrographs, ['lowRes', 'highRes'])]
+
+    """
+
+    _targets = []
+    _environments = [DESKTOP_TKINTER] # This can be ignored
+
+    def show(self, form, *params):
+        """
+        EMPTY METHOD. Needs to be implemented in your class. This will be called to show the wizard.
+
+        :param form: the protocol form, given access to to all parameters. Some times the same wizard will modify several elements in the form.
+        :param params: a list of params to modify. Sometimes the wizard can be generic and can be used for different parameters in the same form.
+        """
+        pass
+    
+    def getView(self):
+        """
+        EMPTY METHOD. Deprecated.This method should return the string value of the view in web
+        that will respond to this wizard. This method only should be implemented
+        in those wizards that have WEB_DJANGO environment defined. 
+        """
+        return None

pyworkflowtests/__init__.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
+# *
+# * [1] SciLifeLab, Stockholm University
+# *
+# * This program is free software: you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation, either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+Mock Domain definition that will be used by basic tests.
+"""
+
+from pyworkflow.protocol import Protocol
+from pyworkflow.wizard import Wizard
+from pyworkflow.viewer import Viewer
+from pyworkflow.utils import getSubclasses
+from pyworkflow.plugin import Domain as pwDomain, Plugin as pwPlugin
+
+from .objects import MockObject
+
+
+class TestDomain(pwDomain):
+    _name = __name__
+    _objectClass = MockObject
+    _protocolClass = Protocol
+    _viewerClass = Viewer
+    _wizardClass = Wizard
+    _baseClasses = getSubclasses(MockObject, globals())
+
+
+Domain = TestDomain
+
+
+class Plugin(pwPlugin):
+    pass

pyworkflowtests/bibtex.py

@@ -0,0 +1,51 @@
+# coding: latin-1
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
+# *
+# * [1] SciLifeLab, Stockholm University
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+
+@article{delaRosaTrevin201693,
+title = "Scipion: A software framework toward integration, reproducibility and validation in 3D electron microscopy ",
+journal = "Journal of Structural Biology",
+volume = "195",
+number = "1",
+pages = "93 - 99",
+year = "2016",
+note = "",
+issn = "1047-8477",
+doi = "http://doi.org/10.1016/j.jsb.2016.04.010",
+url = "http://www.sciencedirect.com/science/article/pii/S104784771630079X",
+author = "J.M. de la Rosa-Trev�n and A. Quintana and L. del Cano and A. Zald�var and I. Foche and J. Guti�rrez and
+J. G�mez-Blanco and J. Burguet-Castell and J. Cuenca-Alba and V. Abrishami and J. Vargas and J. Ot�n and G. Sharov
+and J.L. Vilas and J. Navas and P. Conesa and M. Kazemi and R. Marabini and C.O.S. Sorzano and J.M. Carazo",
+keywords = "Electron microscopy",
+keywords = "Single particle analysis",
+keywords = "Image processing",
+keywords = "Software package",
+keywords = "Workflows",
+keywords = "Reproducibility "
+}
+
+"""