scipion-pyworkflow 3.11.0__py3-none-any.whl → 3.11.1__py3-none-any.whl

pyworkflow/utils/which.py

@@ -0,0 +1,229 @@
+
+# Copyright (c) 2002-2005 ActiveState Corp.
+# See LICENSE.txt for license details.
+# Author:
+#   Trent Mick (TrentM@ActiveState.com)
+# Home:
+#   http://trentm.com/projects/which/
+
+"""Find the full path to commands.
+
+which(command, path=None, verbose=0, exts=None)
+    Return the full path to the first match of the given command on the
+    path.
+
+whichall(command, path=None, verbose=0, exts=None)
+    Return a list of full paths to all matches of the given command on
+    the path.
+
+whichgen(command, path=None, verbose=0, exts=None)
+    Return a generator which will yield full paths to all matches of the
+    given command on the path.
+    
+"""
+
+import os
+import sys
+import stat
+# ---- exceptions
+
+
+class WhichError(Exception):
+    pass
+
+# ---- internal support stuff
+
+
+def _getRegisteredExecutable(exeName):
+    """Windows allow application paths to be registered in the registry."""
+    registered = None
+    if sys.platform.startswith('win'):
+        if os.path.splitext(exeName)[1].lower() != '.exe':
+            exeName += '.exe'
+        import winreg
+        try:
+            key = "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\App Paths\\" +\
+                  exeName
+            value = winreg.QueryValue(winreg.HKEY_LOCAL_MACHINE, key)
+            registered = (value, "from HKLM\\"+key)
+        except winreg.error:
+            pass
+        if registered and not os.path.exists(registered[0]):
+            registered = None
+    return registered
+
+
+def _samefile(fname1, fname2):
+    if sys.platform.startswith('win'):
+        return (os.path.normpath(os.path.normcase(fname1)) ==
+                os.path.normpath(os.path.normcase(fname2)))
+    else:
+        return os.path.samefile(fname1, fname2)
+
+
+def _cull(potential, matches, verbose=0):
+    """Cull inappropriate matches. Possible reasons:
+        - a duplicate of a previous match
+        - not a disk file
+        - not executable (non-Windows)
+    If 'potential' is approved it is returned and added to 'matches'.
+    Otherwise, None is returned.
+    """
+    for match in matches:  # don't yield duplicates
+        if _samefile(potential[0], match[0]):
+            if verbose:
+                sys.stderr.write("duplicate: %s (%s)\n" % potential)
+            return None
+    else:
+        if not stat.S_ISREG(os.stat(potential[0]).st_mode):
+            if verbose:
+                sys.stderr.write("not a regular file: %s (%s)\n" % potential)
+        elif not os.access(potential[0], os.X_OK):
+            if verbose:
+                sys.stderr.write("no executable access: %s (%s)\n"
+                                 % potential)
+        else:
+            matches.append(potential)
+            return potential
+
+        
+# ---- module API
+
+def whichgen(command, path=None, verbose=0, exts=None):
+    """Return a generator of full paths to the given command.
+    
+    :param command: is a the name of the executable to search for.
+    :param path: is an optional alternate path list to search. The default it
+        to use the PATH environment variable.
+    :param verbose: if true, will cause a 2-tuple to be returned for each
+        match. The second element is a textual description of where the
+        match was found.
+    :param exts: optionally allows one to specify a list of extensions to use
+        instead of the standard list for this system. This can
+        effectively be used as an optimization to, for example, avoid
+        stat's of "foo.vbs" when searching for "foo" and you know it is
+        not a VisualBasic script but ".vbs" is on PATHEXT. This option
+        is only supported on Windows.
+
+    :return: This method returns a generator which yields either full paths to
+        the given command or, if verbose, tuples of the form (<path to
+        command>, <where path found>).
+
+    """
+    matches = []
+    if path is None:
+        usingGivenPath = 0
+        path = os.environ.get("PATH", "").split(os.pathsep)
+        if sys.platform.startswith("win"):
+            path.insert(0, os.curdir)  # implied by Windows shell
+    else:
+        usingGivenPath = 1
+
+    # Windows has the concept of a list of extensions (PATHEXT env var).
+    if sys.platform.startswith("win"):
+        if exts is None:
+            exts = os.environ.get("PATHEXT", "").split(os.pathsep)
+            # If '.exe' is not in exts then obviously this is Win9x and
+            # or a bogus PATHEXT, then use a reasonable default.
+            for ext in exts:
+                if ext.lower() == ".exe":
+                    break
+            else:
+                exts = ['.COM', '.EXE', '.BAT']
+        elif not isinstance(exts, list):
+            raise TypeError("'exts' argument must be a list or None")
+    else:
+        if exts is not None:
+            raise WhichError("'exts' argument is not supported on "
+                             "platform '%s'" % sys.platform)
+        exts = []
+
+    # File name cannot have path separators because PATH lookup does not
+    # work that way.
+    if os.sep in command or os.altsep and os.altsep in command:
+        pass
+    else:
+        for i in range(len(path)):
+            dirName = path[i]
+            # On windows the dirName *could* be quoted, drop the quotes
+            if sys.platform.startswith("win") and len(dirName) >= 2\
+               and dirName[0] == '"' and dirName[-1] == '"':
+                dirName = dirName[1:-1]
+            for ext in ['']+exts:
+                absName = os.path.abspath(
+                    os.path.normpath(os.path.join(dirName, command+ext)))
+                if os.path.isfile(absName):
+                    if usingGivenPath:
+                        fromWhere = "from given path element %d" % i
+                    elif not sys.platform.startswith("win"):
+                        fromWhere = "from PATH element %d" % i
+                    elif i == 0:
+                        fromWhere = "from current directory"
+                    else:
+                        fromWhere = "from PATH element %d" % (i-1)
+                    match = _cull((absName, fromWhere), matches, verbose)
+                    if match:
+                        if verbose:
+                            yield match
+                        else:
+                            yield match[0]
+        match = _getRegisteredExecutable(command)
+        if match is not None:
+            match = _cull(match, matches, verbose)
+            if match:
+                if verbose:
+                    yield match
+                else:
+                    yield match[0]
+
+
+def which(command, path=None, verbose=0, exts=None):
+    """Return the full path to the first match of the given command on
+    the path.
+
+    :param command: is a the name of the executable to search for.
+    :param path: is an optional alternate path list to search.
+        The default is to use the PATH environment variable.
+    :param verbose: if true, will cause a 2-tuple to be returned.
+        The second element is a textual description of where the match was found.
+    :param exts: optionally allows one to specify a list of extensions to use
+        instead of the standard list for this system. This can
+        effectively be used as an optimization to, for example, avoid
+        stat's of "foo.vbs" when searching for "foo" and you know it is
+        not a VisualBasic script but ".vbs" is on PATHEXT. This option
+        is only supported on Windows.
+
+    :return If no match is found for the command, an empty string is returned.
+
+    """
+    try:
+        match = next(whichgen(command, path, verbose, exts))
+    except StopIteration:
+        return ''
+    return match
+
+
+def commandExists(command):
+    path = which(command)
+    return path != ''
+
+
+def whichall(command, path=None, verbose=0, exts=None):
+    """Return a list of full paths to all matches of the given command
+    on the path.  
+
+    :param command: is a the name of the executable to search for.
+    :param path: is an optional alternate path list to search. The default it
+        to use the PATH environment variable.
+    :param verbose: if true, will cause a 2-tuple to be returned for each
+        match. The second element is a textual description of where the
+        match was found.
+    :param exts: optionally allows one to specify a list of extensions to use
+        instead of the standard list for this system. This can
+        effectively be used as an optimization to, for example, avoid
+        stat's of "foo.vbs" when searching for "foo" and you know it is
+        not a VisualBasic script but ".vbs" is on PATHEXT. This option
+        is only supported on Windows.
+
+    """
+    return list(whichgen(command, path, verbose, exts))

pyworkflow/webservices/__init__.py

@@ -0,0 +1,8 @@
+
+# Export all configuration variables in case they will be needed outside this module
+# Nonetheless, the recommend way is to import the classes or functions provided
+# by the module
+from .config import *
+
+from .notifier import ProjectWorkflowNotifier
+from .repository import WorkflowRepository

pyworkflow/webservices/config.py

@@ -0,0 +1,8 @@
+# Variable related to the online services (workflow stats and repository) that Scipion reports to or communicate.
+# These variables may change between Scipion versions but do not depend on the user or system
+
+
+# Web that handles workflows
+WORKFLOW_REPOSITORY_SERVER = 'https://workflows.scipion.i2pc.es/'
+WORKFLOW_PROG_STEP1 = 'workflowProgStep1_add/'
+WORKFLOW_PROG_STEP2 = 'workflowProgStep2_add/'

pyworkflow/webservices/notifier.py

@@ -0,0 +1,152 @@
+# **************************************************************************
+# *
+# * Authors:    Roberto Marabini       (roberto@cnb.csic.es)
+#               J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+
+import os
+import time
+import threading
+import uuid
+from datetime import timedelta, datetime
+from urllib.parse import urlencode
+from urllib.request import build_opener, HTTPHandler
+
+import pyworkflow.utils as pwutils
+from pyworkflow import Config
+
+
+
+class ProjectWorkflowNotifier(object):
+    """ Implement different types of notifications about a given
+    project. Currently, the protocols in a workflow will be sent.
+    """
+
+    def __init__(self, project):
+        self.project = project
+
+    def _getUuidFileName(self):
+        return self.project.getLogPath("uuid.log")
+
+    def _getDataFileName(self, fileName="data.log"):
+        return self.project.getLogPath(fileName)
+
+    def _getUuid(self):
+        # Load (or create if not exits) a file
+        # in the project Logs folder to store an unique
+        # project identifier
+        uuidFn = self._getUuidFileName()
+        try:
+            with open(uuidFn) as f:
+                uuidValue = f.readline()
+        except IOError:
+            uuidValue = str(uuid.uuid4())
+            with open(uuidFn, 'w') as f:
+                f.write(uuidValue)
+
+        return uuidValue
+
+    def _modifiedBefore(self, seconds):
+        """ Return True if the uuid.log file has been modified within a given
+        number of seconds. """
+        uuidFn = self._getUuidFileName()
+        if not os.path.exists(uuidFn):
+            return False
+        mTime = datetime.fromtimestamp(os.path.getmtime(uuidFn))
+        delta = datetime.now() - mTime
+
+        return delta < timedelta(seconds=seconds)
+
+    def _sendData(self, url, project_workflow):
+        try:
+            # then connect to webserver a send json
+            # set debuglevel=0 for no messages
+
+            dataDict = {'project_uuid': self._getUuid(),
+                        'project_workflow': project_workflow}
+
+            opener = build_opener(HTTPHandler(debuglevel=0))
+            data = urlencode(dataDict).encode()
+            opener.open(url, data=data).read()
+
+            # Store file time stamp with last time it was sent
+            now = time.time()
+            os.utime(self._getUuidFileName(), (now, now))
+
+            # Write what was sent in a file for _modifiedBefore to check file TS and avoid resending stats
+            dataFile = self._getDataFileName()
+            # create the folder of the file path if not exists
+            pwutils.makeFilePath(dataFile)
+            with open(dataFile, 'w') as f:
+                f.write(project_workflow)
+
+        except Exception as e:
+            # Tolerate errors
+            pass
+
+    def _dataModified(self, projectWorfklow):
+        try:
+            with open(self._getDataFileName()) as f:
+                projectWorfklow2 = f.readline()
+                if projectWorfklow2 == projectWorfklow:
+                    return False
+        except IOError:
+            pass
+        return True
+
+    def notifyWorkflow(self):
+
+        try:
+            # check if environment exists otherwise abort
+            if not Config.SCIPION_NOTIFY:
+                return
+
+            # if project specifies not to send stats
+            if self._isProjectMuted():
+                return
+
+            # Check the seconds range of the notify, by default one day
+            seconds = int(os.environ.get('SCIPION_NOTIFY_SECONDS', '86400'))
+
+            if self._modifiedBefore(seconds):  # notify not more than once a day
+                return
+
+            # INFO: now we are only sending the protocols names in the project.
+            # We could pass namesOnly=False to get the full workflow template
+            project_workflow = self.project.getProjectUsage().toJSON()  # self.project.getProtocolsJson(namesOnly=True)
+
+            urlName = Config.SCIPION_STATS_WORKFLOW_APP.strip()
+            urlName += "addOrUpdateWorkflow/"
+            t = threading.Thread(name="notifier", target=lambda: self._sendData(urlName, project_workflow))
+            t.start()  # will execute function in a separate thread
+        except Exception as e:
+            print("Can't report usage: ", e)
+
+    def _isProjectMuted(self):
+        """ Projects are muted if they come from tests, Since there is no flag for it
+        we will assume that if the project name starts with Test it will be considered
+        a test and therefore no statistics will be sent"""
+        return os.path.basename(self.project.name).startswith("Test")
+

pyworkflow/webservices/repository.py

@@ -0,0 +1,59 @@
+import webbrowser
+import json
+import requests
+
+from pyworkflow import CORE_VERSION
+from . import config
+
+
+class WorkflowRepository(object):
+    """ Manager to communicate with the workflow repository services.
+    It will provide functions to:
+    - Search workflows (open the url in a browser).
+    - Upload a given workflow json file.
+    """
+    def __init__(self,
+                 repositoryUrl=config.WORKFLOW_REPOSITORY_SERVER,
+                 uploadFileSuffix=config.WORKFLOW_PROG_STEP1,
+                 uploadMdSuffix=config.WORKFLOW_PROG_STEP2):
+        self._url = repositoryUrl
+        self._uploadFileUrl = repositoryUrl + uploadFileSuffix
+        self._uploadMdUrl = repositoryUrl + uploadMdSuffix
+
+    def search(self):
+        """ Open the repository URL in a web browser. """
+        webbrowser.open(self._url)
+
+    def upload(self, jsonFileName):
+        """ Upload a given workflow providing the path ot the json file.
+
+        First the file is uploaded, then the metadata is uploaded.
+        The script uploads the file and then opens a browser for the metadata
+        Note that the two steps are needed since no initial value can be passed
+        to a file field. poster3 module is needed. Poster3 is pure python
+        so it may be added to the directory rather than installed if needed.
+
+        The server is django a uses filefield and csrf_exempt.
+        csrf_exempt disable csrf checking. filefield
+        """
+
+        # we are going to upload a file so this is a multipart
+        # connection
+        with open(jsonFileName, "rb") as workflowFile:
+            file_dict = {"json": workflowFile}
+            response = requests.post(self._uploadFileUrl, files=file_dict)
+
+        # server returns  a json stored as text at response.text
+        _dict = json.loads(response.text)
+
+        version = CORE_VERSION
+        # version hack end
+
+        fnUrl = "?jsonFileName=%s&versionInit=%s" % (_dict['jsonFileName'],
+                                                     version)  # use GET
+        # open browser to fill metadata, fileName will be saved as session
+        # variable. Note that I cannot save the file never in the
+        # session in the first connection because the browser changes
+        # from urlib2 to an actual browser
+        # so sessions are different
+        webbrowser.open(self._uploadMdUrl + fnUrl)

pyworkflow/webservices/workflowhub.py

@@ -0,0 +1,74 @@
+import json
+from urllib.request import Request, urlopen
+from pyworkflow.template import Template
+
+
+class WHTemplate(Template):
+
+    def __init__(self, source, name, description, url):
+        super().__init__(source, name, description)
+        self.url = url
+
+    def loadContent(self):
+        """ Download the file pointer by url and read the content"""
+
+        return make_request(self.url, asJson=False)
+
+
+def make_request(url, asJson=True):
+    """ Makes a request to the url and returns the json as a dictionary"""
+
+    req = Request(url)
+    req.add_header("accept", "application/json")
+
+    with urlopen(req, timeout=1) as response:
+        if asJson:
+            data = json.load(response)
+        else:
+            html_response = response.read()
+            encoding = response.headers.get_content_charset('utf-8')
+            data = html_response.decode(encoding)
+        return data
+
+
+def get_workflow_file_url(workflow_id, version):
+    root_url = "https://workflowhub.eu/workflows/%s/git/%s/" % (workflow_id, version)
+    url = root_url + 'tree'
+
+    result = make_request(url)
+
+    for file in result["tree"]:
+        path = (file["path"])
+        if path.endswith(".json.template"):
+            return root_url + 'raw/' + path
+
+
+def get_wh_templates(template_id=None, organization="Scipion%20CNB"):
+    """ Returns a list of scipion templates available in workflow hub"""
+
+    url = "https://workflowhub.eu/ga4gh/trs/v2/tools?organization=%s" % organization
+
+    response = make_request(url)
+
+    template_list = []
+
+    for workflow in response:
+        workflow_id = workflow["id"]
+        name = workflow["name"]
+        description = workflow['description']
+        version = workflow["versions"][-1]
+        version_id = version["id"]
+        template_url = get_workflow_file_url(workflow_id, version_id)
+
+        new_template = WHTemplate("Workflow hub", name, description, template_url)
+        if template_id is None or new_template.getObjId() == template_id:
+            template_list.append(new_template)
+
+    return template_list
+
+
+if __name__ == "__main__":
+
+    templates = get_wh_templates()
+    for template in templates:
+        print(template)

pyworkflowtests/tests/test_canvas.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# To run only the tests in this file, use:
+# python -m unittest test_canvas -v
+
+import pyworkflow.gui.canvas
+import tkinter
+import math
+from pyworkflow.tests import *
+
+
+class TestCanvas(BaseTest):
+    # IMPORTANT: Tk requires at least that DISPLAY is defined
+    # hence in some environments (like buildbot) the test may fail,
+    # check for the TclError exception
+
+    _labels = [PULL_REQUEST]
+
+    @classmethod
+    def setUpClass(cls):
+        setupTestOutput(cls)
+
+    def distance(self, c1, c2):
+        return round(math.hypot(c2[0] - c1[0], c2[1] - c1[1]), 2)
+
+    def allEqual(self, values):
+        return not values or values.count(values[0]) == len(values)
+
+    def allDifferent(self, values):
+        return not values or len(values) == len(set(values))
+
+    def test_connectorsCoords(self):
+        try:
+            root = tkinter.Tk()
+            canvas = pyworkflow.gui.canvas.Canvas(root, width=800, height=600)
+            tb1 = canvas.createTextbox("First", 100, 100, "blue")
+
+            connectorsCoords = tb1.getConnectorsCoordinates()
+            self.assertTrue(self.allDifferent(connectorsCoords))
+
+            print(connectorsCoords)
+
+            distances = {}
+            for i in range(len(connectorsCoords) - 1):
+                distances[i] = self.distance(connectorsCoords[i],
+                                             connectorsCoords[i + 1])
+
+                print(distances)
+                self.assertTrue(self.allEqual(list(distances.values())))
+                self.assertNotEqual(distances[0], 0)
+        except tkinter.TclError as ex:
+            print(ex)
+
+    def test_closestConnectors(self):
+        try:
+            root = tkinter.Tk()
+            canvas = pyworkflow.gui.canvas.Canvas(root, width=800, height=600)
+            tb1 = canvas.createTextbox("Textbox1", 100, 100, "blue")
+            tb2 = canvas.createTextbox("Textbox2", 300, 100, "blue")
+            tb3 = canvas.createTextbox("Textbox3", 100, 300, "blue")
+
+            tb1ConnectorsCoords = tb1.getConnectorsCoordinates()
+            tb2ConnectorsCoords = tb2.getConnectorsCoordinates()
+            tb3ConnectorsCoords = tb3.getConnectorsCoordinates()
+            c1, c2 = pyworkflow.gui.canvas.findStrictClosestConnectors(tb1, tb2)
+            c3, c4 = pyworkflow.gui.canvas.findStrictClosestConnectors(tb1, tb3)
+            # tb1 and tb2 are aligned vertically. tb1 and tb3, horizontally.
+            # So, their closest connectors must share one coordinate (y in case of c1&c2, x i n case of c3&c4)
+            self.assertEqual(c1[1], c2[1])
+            self.assertEqual(c3[0], c4[0])
+
+        except tkinter.TclError as ex:
+            print(ex)

pyworkflowtests/tests/test_domain.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
+# *
+# * [1] SciLifeLab, Stockholm University
+# *
+# * This program is free software: you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation, either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+import pyworkflow.tests as pwtests
+import pyworkflowtests.objects as objectsMod
+from pyworkflowtests import Domain
+
+
+class TestDomain(pwtests.BaseTest):
+
+    def test_objects(self):
+        """ Test that all objects are properly discovered. """
+        objects = Domain.getObjects()
+        for k in dir(objectsMod):
+            v = getattr(objectsMod, k)
+            if isinstance(v, objectsMod.MockObject):
+                self.assertEqual(objects[k], v)
+
+    def test_viewers(self):
+        pass
+
+    def test_wizards(self):
+        pass