scipion-pyworkflow 3.10.5__py3-none-any.whl → 3.11.0__py3-none-any.whl

pyworkflow/utils/dataset.py

@@ -1,414 +0,0 @@
-# **************************************************************************
-# *
-# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
-# *
-# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
-# *
-# * This program is free software; you can redistribute it and/or modify
-# * it under the terms of the GNU General Public License as published by
-# * the Free Software Foundation; either version 3 of the License, or
-# * (at your option) any later version.
-# *
-# * This program is distributed in the hope that it will be useful,
-# * but WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# * GNU General Public License for more details.
-# *
-# * You should have received a copy of the GNU General Public License
-# * along with this program; if not, write to the Free Software
-# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# * 02111-1307  USA
-# *
-# *  All comments concerning this program package may be sent to the
-# *  e-mail address 'scipion@cnb.csic.es'
-# *
-# **************************************************************************
-"""
-# JMRT (2018-12-11) This module is almost not used at all. Maybe it
-can be removed in a future. Just kept here for the moment inside
-pw.utils and not imported by default
-"""
-
-import os
-from collections import OrderedDict, namedtuple
-
-from pyworkflow.mapper import SqliteFlatDb, SqliteDb
-
-
-class DataSet(object):
-    """ Holds several Tables
-    All tables should have an unique tableName. 
-    """
-    
-    def __init__(self, tables, tableName=None, volumeName=None, numberSlices=0):
-        self._tables = list(tables)
-        self._tableName = tableName
-        # FIXME: You have to see if the volumeName is used anywhere
-        self._volumeName = volumeName 
-        self._numberSlices = numberSlices 
-        self.projectPath = None
-        
-    def currentTable(self):
-        """ Returns the name of the last selected table. """    
-        return self._tableName
-        
-    def setVolumeName(self, volumeName):    
-        self._volumeName = volumeName
-        
-    def getVolumeName(self):
-        return self._volumeName
-    
-    def setNumberSlices(self, numberSlices):
-        self._numberSlices = numberSlices
-        
-    def getNumberSlices(self):
-        return self._numberSlices    
-    
-    def getNumberSlicesForTemplate(self):
-        return range(self._numberSlices)
-        
-    def listTables(self):
-        """ List the actual table names on the DataSet. """
-        return self._tables
-    
-    def getTable(self, tableName=None):
-        if tableName is None:
-            tableName = self.listTables()[0]
-        if tableName not in self._tables:
-            raise Exception("DataSet: table '%s' not found.\n   Current tables: %s" % 
-                            (tableName, self._tables))
-            
-        table = self._loadTable(tableName)
-        self._tableName = tableName
-        
-        return table
-    
-    def getTypeOfColumn(self, label):
-        """ this method should be implemented by subclasses. """
-        pass
-    
-    def _loadTable(self, tableName):
-        """ this method should be implemented by subclasses. """
-        pass
-
-
-class Table(object):
-    """ Table to hold rows of data. 
-    A table contains a list of columns. """
-    def __init__(self, *columns):
-        self._columns = OrderedDict()
-        self._rowDict = OrderedDict()
-        self._addColumn(Column('id', int))
-        
-        for col in columns:
-            self._addColumn(col)
-            
-        colNames = [col.getName() for col in self.iterColumns()]
-        # This imply that columns can only be added in __init__
-        self.Row = namedtuple('Row', colNames)
-    
-    def setLabelToRender(self, labelToRender):
-        self._labelToRender = labelToRender
-        
-    def _addColumn(self, col):
-        self._columns[col.getName()] = col
-        
-    def getColumnValues(self, columnName):
-        if self.hasColumn(columnName):
-            return [getattr(row, columnName) for row in self.iterRows()] 
-        else:
-            return [None] * self.getSize()
-        
-    def iterColumns(self):
-        return self._columns.values()
-    
-    def hasColumn(self, columnName):
-        """ Return true if column exists """
-        return columnName in self._columns
-    
-    def getColumn(self, columnName):
-        if columnName not in self._columns:
-            raise Exception('Table: column "%s" not found.\nCurrent columns: %s' % (
-                columnName, '\n'.join(self._columns.keys())))
-        return self._columns[columnName] 
-    
-    def hasEnabledColumn(self):
-        """ Return true if enabled column exists """
-        return self.hasColumn('enabled')
-        
-    def getColumns(self):
-        """ Return all columns. """
-        return self._columns.values()
-    
-    def getNumberOfColumns(self):
-        return len(self._columns)
-    
-    def getSize(self):
-        """ Return the number of rows. """
-        return len(self._rowDict)
-    
-    def getRows(self):
-        """ Return all rows. """
-        return [row for row in self.iterRows()]
-    
-    def getRow(self, rowId):
-        return self._rowDict[rowId]
-    
-    def _setRow(self, rowId, row):
-        self._rowDict[rowId] = row
-    
-    def getDataToRenderAndExtra(self):
-        return zip(self.getIdColumn(),
-                   self.getColumnValues("enabled"),
-                   self.getDataToRender(),
-                   self.getTransformationMatrix())
-    
-    def getDataToRender(self):
-        return self.getColumnValues(self._labelToRender)
-    
-    def getIdColumn(self):
-        return self.getColumnValues("id")
-        
-    def getTransformationMatrix(self):    
-        return self.getColumnValues(self._labelToRender+"_transformationMatrix")
-    
-    def _convertValues(self, values):
-        """ Convert the input values to the actual
-        expected type of each column. 
-        """
-        cValues = {}
-        for k, v in values.items():
-            col = self.getColumn(k)
-            cValues[k] = col.convert(v)
-        
-        return cValues
-        
-    def addRow(self, rowId, **values):
-        """ With this implementation the rowId should be provided.
-        We need to work around to also allow automatic generation of id's
-        """
-        values['id'] = rowId
-        
-        for col in self.iterColumns():
-            if col.getName() not in values:
-                if col.hasDefault():
-                    values[col.getName()] = col.getDefault()
-                else:
-                    raise Exception('Table: value for column "%s" not provided.' % col.getName())
-                
-        row = self.Row(**self._convertValues(values))
-        self._setRow(rowId, row)
-        
-    def updateRow(self, rowId, **values):
-        """ Update a row given its rowId and some values to update. """
-        row = self.getRow(rowId)
-        self._setRow(rowId, row._replace(**self._convertValues(values)))
-    
-    def iterRows(self):
-        """ Iterate over the rows. """
-        return self._rowDict.values()
-    
-    def getValueFromIndex(self, index, label):
-        """ Return the value of the property 'label'
-        in the element that has this 'index'.
-        """
-        value = list(self._rowDict.values())[index]._asdict()[label]
-        return value
-    
-    def getIndexFromValue(self, value, label):
-        """ Search the element that has property 'label'
-        equals to value and returns its index.
-        """
-        for index, row in enumerate(self.iterRows()):
-            if value == row._asdict()[label]:
-                return index            
-        return -1
-    
-    def __str__(self):
-        return '\n'.join([str(row) for row in self.iterRows()])
-    
-
-# JMRT (2018-12-11) This constants are duplicated in showj, since this module
-# is not widely used I don't find convenient such a dependency
-COL_RENDER_NONE = 0
-COL_RENDER_ID = 1
-COL_RENDER_TEXT = 2
-COL_RENDER_IMAGE = 3
-COL_RENDER_CHECKBOX = 4
-COL_RENDER_VOLUME = 5
-
-
-class Column(object):
-    def __init__(self, colName, colType=None, default=None, 
-                 label=None, renderType=COL_RENDER_NONE):
-        self._name = colName
-        self._type = colType
-        self._default = default
-        self._label = label or colName
-        self._renderType = renderType
-        
-    def getName(self):
-        return self._name
-
-    def getLabel(self):
-        return self._label
-        
-    def getType(self):
-        return self._type
-    
-    def convert(self, value):
-        """ Try to convert the value to the column type. """
-        return self._type(value)
-    
-    def hasDefault(self):
-        return self._default is not None
-    
-    def getDefault(self):
-        return self._default
-    
-    def getRenderType(self):
-        return self._renderType
-    
-    def setRenderType(self, renderType):
-        self._renderType = renderType
-    
-
-class SqliteDataSet(DataSet):
-    """ Provide a DataSet implementation based on sqlite file.
-    The tables of the dataset will be the object tables in database.
-    Each block is a table on the dataset. 
-    """
-    
-    def __init__(self, filename):
-        self._dbName = filename
-        db = SqliteDb()
-        db._createConnection(filename, 1000)
-        # Tables should be at pairs:
-        # PREFIX_Classes
-        # PREFIX_Objects  
-        # where PREFIX can be empty
-        self.tablePrefixes = OrderedDict()
-        tables = db.getTables()
-        for t in tables:
-            if t.endswith('Classes'):
-                prefix = t.replace('Classes', '')
-                to = prefix + 'Objects'
-                if to not in tables:
-                    raise Exception('SqliteDataSet: table "%s" found, but not "%s"' % (t, to))
-                flatDb = SqliteFlatDb(filename, tablePrefix=prefix)
-                tableName = prefix + self._getPlural(flatDb.getSelfClassName())
-                self.tablePrefixes[tableName] = prefix
-                # tablePrefixes.append(prefix)
-        DataSet.__init__(self, self.tablePrefixes.keys())
-        db.close()
-        
-    def _getPlural(self, className):
-        """ Get the plural of word for tables labels. """
-        if className.startswith('Class'):
-            return className.replace('Class', 'Classes')
-        return className + 's'
-        
-    def _loadTable(self, tableName):
-        """ Load information from tables PREFIX_Classes, PREFIX_Objects. """
-        
-        tableName = self.tablePrefixes[tableName]
-        
-        BASIC_COLUMNS = [Column('id', int, renderType=COL_RENDER_ID), 
-                         Column('enabled', bool, renderType=COL_RENDER_CHECKBOX),
-                         Column('label', str), 
-                         Column('comment', str),
-                         Column('creation', str)]
-        # Load columns from PREFIX_Classes table
-        columns = list(BASIC_COLUMNS)
-        db = SqliteDb()
-        db._createConnection(self._dbName, 1000)
-        db.executeCommand("SELECT * FROM %sClasses;" % tableName)
-        # This will store the images columns to join
-        # the _index and the _filename
-        imgCols = {}
-        for row in db._iterResults():
-            renderType = COL_RENDER_NONE
-            colName = row['column_name']
-            colLabel = row['label_property']
-            
-            if colLabel != 'self':
-                # Keep track of _index and _filename pairs to mark as renderable images
-                if colLabel.endswith('_index'):
-                    imgCols[colLabel.replace('_index', '')] = colName
-                
-                elif colLabel.endswith('_filename'):
-                    
-                    # TODO: Maybe not all the labels endswith "_filename" 
-                    # have to be rendered. 
-                    # for example in the RotSpectra with '_representative._filename'
-
-                    prefix = colLabel.replace('_filename', '')
-                    if prefix in imgCols:
-                        renderType = COL_RENDER_IMAGE
-                        imgCols[colName] = imgCols[prefix]
-                
-                # CTF FIX
-                elif (colLabel.endswith('_psdFile') or 
-                      colLabel.endswith('_enhanced_psd') or 
-                      colLabel.endswith('_ctfmodel_quadrant') or 
-                      colLabel.endswith('_ctfmodel_halfplane')):
-                    
-                    renderType = COL_RENDER_IMAGE
-                
-                if row['class_name'] == 'Boolean':
-                    renderType = COL_RENDER_CHECKBOX   
-                columns.append(Column(colName, str, label=colLabel, renderType=renderType))
-        table = Table(*columns)
-        
-        checkedImgCols = {}  # Check if the image columns are volumes
-        # FIXME: Move this to scipion-em? Maybe remove the whole module that is not used?
-        from pwem.emlib.image import ImageHandler
-        ih = ImageHandler() 
-        
-        # Populate the table in the DataSet
-        db.executeCommand("SELECT * FROM %sObjects;" % tableName)
-        for row in db._iterResults():
-            rowDict = dict(row)
-            for k, v in rowDict.items():
-                if v is None:
-                    rowDict[k] = ''
-                # Set the index@filename for images columns values
-                if k in imgCols:
-                    colName = imgCols[k]
-                    index = rowDict[colName]
-                    
-                    filename = os.path.join(self.projectPath, rowDict[k])
-                    filepath = filename.replace(":mrc", "")
-                    if not checkedImgCols.get(colName, False):
-                        if os.path.exists(filepath):
-                            # print "Fn to get dims: %s@%s" % (index,filename)
-                            x, y, z, n = ih.getDimensions((index, filename))
-                            if z > 1:
-                                table.getColumn(k).setRenderType(COL_RENDER_VOLUME)
-                        checkedImgCols[colName] = True
-                    if index:
-                        rowDict[k] = '%06d@%s' % (index, filename)
-            table.addRow(row['id'], **rowDict)
-            
-        return table
-        
-        
-class SingleFileDataSet(DataSet):
-    """ DataSet implementation for single files such as Images or Volumes. 
-    """
-    
-    def __init__(self, filename):
-        self._filename = filename
-        self._tableName = ""
-        DataSet.__init__(self, [self._tableName])
-        self._table = self._createSingleTable()
-        
-    def _createSingleTable(self):
-        table = Table(Column('filename', str, 
-                             renderType=COL_RENDER_VOLUME))  # FIXME: for single images we need to read the dimensions
-        table.addRow(1, filename=self._filename)
-        
-        return table
-        
-    def _loadTable(self, tableName):
-        return self._table

pyworkflow/utils/echo.py

@@ -1,104 +0,0 @@
-# **************************************************************************
-# *
-# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
-# *
-# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
-# *
-# * This program is free software; you can redistribute it and/or modify
-# * it under the terms of the GNU General Public License as published by
-# * the Free Software Foundation; either version 3 of the License, or
-# * (at your option) any later version.
-# *
-# * This program is distributed in the hope that it will be useful,
-# * but WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# * GNU General Public License for more details.
-# *
-# * You should have received a copy of the GNU General Public License
-# * along with this program; if not, write to the Free Software
-# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# * 02111-1307  USA
-# *
-# *  All comments concerning this program package may be sent to the
-# *  e-mail address 'scipion@cnb.csic.es'
-# *
-# **************************************************************************
-"""
-This module trace through printing 
-module and class functions...
-Code from: http://wordaligned.org/articles/echo
-"""
-
-
-import inspect
-import sys
-    
-
-def name(item):
-    """ Return an item's name. """
-    return item.__name__
-
-
-def is_classmethod(instancemethod):
-    """ Determine if an instancemethod is a classmethod. """
-    return instancemethod.im_self is not None
-
-
-def is_class_private_name(name):
-    """ Determine if a name is a class private name. """
-    # Exclude system defined names such as __init__, __add__ etc
-    return name.startswith("__") and not name.endswith("__")
-
-
-def method_name(method):
-    """ Return a method's name.
-    
-    This function returns the name the method is accessed by from
-    outside the class (i.e. it prefixes "private" methods appropriately).
-    """
-    mname = name(method)
-    if is_class_private_name(mname):
-        mname = "_%s%s" % (name(method.im_class), mname)
-    return mname
-
-
-def format_args(args):
-    print(args)
-
-
-def echo(fn, write=sys.stdout.write):
-    print(fn)
-
-
-def echo_instancemethod(klass, method, write=sys.stdout.write):
-    """ Change an instancemethod so that calls to it are echoed.
-    
-    Replacing a classmethod is a little more tricky.
-    See: http://www.python.org/doc/current/ref/types.html
-    """
-    mname = method_name(method)
-    never_echo = "__str__", "__repr__",  # Avoid recursion printing method calls
-    if mname in never_echo:
-        pass
-    elif is_classmethod(method):
-        setattr(klass, mname, classmethod(echo(method.im_func, write)))
-    else:
-        setattr(klass, mname, echo(method, write))
-
-
-def echo_class(klass, write=sys.stdout.write):
-    """ Echo calls to class methods and static functions
-    """
-    for _, method in inspect.getmembers(klass, inspect.ismethod):
-        echo_instancemethod(klass, method, write)
-    for _, fn in inspect.getmembers(klass, inspect.isfunction):
-        setattr(klass, name(fn), staticmethod(echo(fn, write)))
-
-
-def echo_module(mod, write=sys.stdout.write):
-    """ Echo calls to functions and methods in a module.
-    """
-    for fname, fn in inspect.getmembers(mod, inspect.isfunction):
-        setattr(mod, fname, echo(fn, write))
-    for _, klass in inspect.getmembers(mod, inspect.isclass):
-        echo_class(klass, write)

pyworkflow/utils/graph.py

@@ -1,169 +0,0 @@
-# **************************************************************************
-# *
-# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
-# *
-# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
-# *
-# * This program is free software; you can redistribute it and/or modify
-# * it under the terms of the GNU General Public License as published by
-# * the Free Software Foundation; either version 3 of the License, or
-# * (at your option) any later version.
-# *
-# * This program is distributed in the hope that it will be useful,
-# * but WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# * GNU General Public License for more details.
-# *
-# * You should have received a copy of the GNU General Public License
-# * along with this program; if not, write to the Free Software
-# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# * 02111-1307  USA
-# *
-# *  All comments concerning this program package may be sent to the
-# *  e-mail address 'scipion@cnb.csic.es'
-# *
-# **************************************************************************
-"""
-This module define a Graph class and some utilities 
-"""
-import logging
-logger = logging.getLogger(__name__)
-
-
-class Node(object):
-    """ A node inside the graph. """
-    _count = 1
-
-    def __init__(self, name=None, label=None):
-        self._children = []
-        self._parents = []
-
-        if name is None:
-            name = str(self._count)
-            self._count += 1
-        self._name = name
-
-        if label is None:
-            label = name
-        self._label = label
-        self._fixed = False
-
-    def getName(self):
-        return self._name
-
-    def getLabel(self):
-        return self._label
-
-    def setLabel(self, newLabel):
-        self._label = newLabel
-
-    def isRoot(self):
-        return len(self._parents) == 0
-
-    def getChildren(self):
-        return self._children
-
-    def addChild(self, *nodes):
-        for n in nodes:
-            if n not in self._children:
-                self._children.append(n)
-                n._parents.append(self)
-
-    def getParent(self):
-        """ Return the first parent in the list,
-        if the node isRoot, None is returned.
-        """
-        if self.isRoot():
-            return None
-
-        return self._parents[0]
-
-    def getParents(self):
-        return self._parents
-
-    def iterChildren(self):
-        """ Iterate over all children and sub-children.
-        Nodes can be visited more than once if it has
-        more than one parent.
-        """
-        for child in self._children:
-            for c in child.iterChildren():
-                yield c
-
-        yield self
-
-    def countChildren(self, visitedNode=None, count=0):
-        """ Iterate over all childs and subchilds.
-        Nodes can be visited once
-        """
-        for child in self._children:
-            if child._name not in visitedNode:
-                visitedNode[child._name] = True
-            child.countChildren(visitedNode)
-        return len(visitedNode)
-
-
-    def iterChildrenBreadth(self):
-        """ Iter child nodes in a breadth-first order
-        """
-        for child in self._children:
-            yield child
-
-        for child in self._children:
-            for child2 in child.iterChildrenBreadth():
-                yield child2
-
-    def __str__(self):
-        return "Node (id=%s, label=%s, root=%s)" % (self._name,
-                                                    self.getLabel(),
-                                                    self.isRoot())
-
-
-class Graph(object):
-    """Simple directed Graph class. 
-    Implemented using adjacency lists.
-    """
-
-    def __init__(self, rootName='ROOT', root=None):
-        self._nodes = []
-        self._nodesDict = {}  # To retrieve nodes from name
-        if root is None:
-            self._root = self.createNode(rootName)
-        else:
-            self._root = root
-            self._registerNode(root)
-
-    def _registerNode(self, node):
-        self._nodes.append(node)
-        self._nodesDict[node.getName()] = node
-        for child in node.getChildren():
-            self._registerNode(child)
-
-    def getRoot(self) -> Node:
-        return self._root
-
-    def createNode(self, nodeName, nodeLabel=None) -> Node:
-        """ Add a node to the graph """
-        node = Node(nodeName, nodeLabel)
-        self._registerNode(node)
-
-        return node
-
-    def aliasNode(self, node, aliasName):
-        """ Register an alias name for the node. """
-        self._nodesDict[aliasName] = node
-
-    def getNode(self, nodeName) -> Node:
-        return self._nodesDict.get(nodeName, None)
-
-    def getNodeNames(self):
-        """ Returns all the keys in the node dictionary"""
-        return self._nodesDict.keys()
-
-    def getNodes(self):
-        return self._nodes
-
-    def getRootNodes(self):
-        """ Return all nodes that have no parent. """
-        return [n for n in self._nodes if n.isRoot()]
-