python-microscopy 25.6.5__cp39-cp39-win_amd64.whl

PYME/IO/FileUtils/readSpeckle.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Jul  9 13:46:29 2015
+
+@author: david
+"""
+import numpy as np
+from PYME.IO import unifiedIO
+
+def readSpeckles(filename):
+    with unifiedIO.openFile(filename, 'r') as f:
+        
+        speckles = []
+        currentSpeckle = None    
+        
+        for l in f.readlines():
+            if l.startswith('#%start speckle'):
+                currentSpeckle = []
+                speckles.append(currentSpeckle)
+            elif l.startswith('#'):
+                #comment
+                pass
+            else:
+                currentSpeckle.append([float(val) for val in l.split('\t')])
+                
+        return [np.array(s) for s in speckles]
+
+
+def gen_traces_from_speckles(speckles, leadFrames=10, followFrames=50, seriesLength=1000000, clipRegion=[0,0,512,512]):
+    """ Generate pseudo particle trajectories from speckle data.
+
+     These trajectories look like they were generated using the normal particle tracking methods, but are centered
+     at the detected speckle position. They extend before the start of the speckle trace (to establish a baseline), and
+     after the end of the speckle trace (to look at the diffusion of the membrane label away from the point of fusion.
+
+    Parameters
+    ----------
+    speckles : list
+        list of [x,y,t]xN arrays containing the trace for each speckle
+    leadFrames : int
+        Number of frames to extract before the start of the speckle for establishing the baseline
+    followFrames : int
+        Number of frames to extract after the end of the speckle to follow the diffusion of the fused dyes.
+    seriesLength : int
+        the length of the image series - traces will not extend beyond the end of the series.
+
+    Returns
+    -------
+    a numpy recarray containing x, y, t, and clumpIndex columns
+    """
+
+    point_dtype = np.dtype([('x_pixels', 'f4'), ('y_pixels', 'f4'), ('t', 'i4'), ('clumpIndex', 'i4')])
+
+    extended_speckles = []
+    
+    x0, y0, x1, y1 = clipRegion
+
+    for i, speck in enumerate(speckles):
+        y_, x_, t_ = speck.T
+        trace_start = int(max(t_[0] - leadFrames, 0))
+        trace_end = int(min(t_[-1] + followFrames, seriesLength))
+
+        xm, ym = x_.mean(), y_.mean()
+        
+        if (xm >= x0) and (ym >= y0) and (xm < x1) and (ym < y1):
+            for j in range(trace_start, trace_end):
+                s_ = np.zeros(1, dtype=point_dtype)
+                s_['x_pixels'] = xm
+                s_['y_pixels'] = ym
+                s_['t'] = j
+                s_['clumpIndex'] = i
+                extended_speckles.append(s_)
+
+    sp = np.hstack(extended_speckles)
+
+    I = np.argsort(sp['t'])
+    return sp[I]
+

PYME/IO/FileUtils/readTiff.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python
+
+##################
+# readTiff.py
+#
+# Copyright David Baddeley, 2009
+# d.baddeley@auckland.ac.nz
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+
+from PIL import Image
+import numpy as np
+from PYME.misc import TiffImagePlugin #monkey patch PIL with improved tiff support from Priithon
+
+
+
+def read3DTiff(filename):
+    im = Image.open(filename)
+
+    im.seek(0)
+
+    #PIL's endedness support is subtly broken - try to fix it
+    #NB this is untested for floating point tiffs
+    endedness = 'LE'
+    if im.ifd.prefix =='MM':
+        endedness = 'BE'
+    
+
+    #ima = np.array(im.getdata(), 'int16').newbyteorder('BE')
+    ima = np.array(im.getdata()).newbyteorder(endedness)
+
+    print((ima.dtype))
+    
+    #print ima.shape
+
+    ima = ima.reshape((im.size[1], im.size[0], 1))
+
+    pos = im.tell()
+
+    try:
+        while True:
+            pos += 1
+            im.seek(pos)
+
+            #ima = np.concatenate((ima, np.array(im.getdata(), 'int16').newbyteorder('BE').reshape((im.size[1], im.size[0], 1))), 2)
+            ima = np.concatenate((ima, np.array(im.getdata()).newbyteorder(endedness).reshape((im.size[1], im.size[0], 1))), 2)
+
+    except EOFError:
+        pass
+
+    return ima

PYME/IO/FileUtils/saveTiffStack.py

@@ -0,0 +1,137 @@
+#!/usr/bin/python
+
+##################
+# saveTiffStack.py
+#
+# Copyright David Baddeley, 2009
+# d.baddeley@auckland.ac.nz
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+
+from PIL import Image
+from PYME.misc import TiffImagePlugin #monkey patch PIL with improved tiff support from Priithon
+import subprocess
+
+def writeTiff(im, outfile):
+    command = ["tiffcp"]
+    # add options here, if any (e.g. for compression)
+
+    #im = im.astype('uint16')
+    im = im.astype('>u2').astype('<u2')
+
+    for i in range(im.shape[2]):
+        framefile = "/tmp/frame%d.tif" % i
+
+        Image.fromarray(im[:,:,i].squeeze(), 'I;16').save(framefile)
+        command.append(framefile)
+
+    command.append(outfile)
+    subprocess.call(command)
+
+    # remove frame files here
+    subprocess.call('rm /tmp/frame*.tif', shell=True)
+
+
+#Adapted to use 3rd dimension as z from priithons 'useful.py'
+def saveTiffMultipage(arr, fn, **params):
+    if arr.ndim == 2:
+        #fake a 3rd dimension
+        #raise ValueError, "can only save 3d arrays"
+        arr = arr[:,:,None]
+
+    fp = open(fn, 'w+b')
+
+    ifd_offsets=[]
+
+    if arr.dtype == 'uint16':
+        nptype = 'uint16'
+        piltype = 'I;16'
+    #elif arr.dtype == 'uint8':
+    #    nptype = 'uint8'
+    #    piltype = 'I;8'
+    else:
+        nptype = 'f'
+        piltype = 'F'
+
+    params["_debug_multipage"] = True
+    for z in range(arr.shape[2]):
+        ii = Image.fromarray(arr[:,:,z].astype(nptype), piltype)
+
+        fp.seek(0,2) # go to end of file
+        if z==0:
+            # ref. PIL  TiffImagePlugin
+            # PIL always starts the first IFD at offset 8
+            ifdOffset = 8
+        else:
+            ifdOffset = fp.tell()
+
+        ii.save(fp, format="TIFF", **params)
+
+        if z>0: # correct "next" entry of previous ifd -- connect !
+            ifdo = ifd_offsets[-1]
+            fp.seek(ifdo)
+            ifdLength = ii._debug_multipage.i16(fp.read(2))
+            fp.seek(ifdLength*12,1) # go to "next" field near end of ifd
+            fp.write(ii._debug_multipage.o32( ifdOffset ))
+
+        ifd_offsets.append(ifdOffset)
+    fp.close()
+
+class TiffMP(object):
+    def __init__(self, fn, **params):
+        self.fp = open(fn, 'w+b')    
+        self.ifd_offsets=[]
+        params["_debug_multipage"] = True
+        
+        self.params = params
+        self.z = 0
+    
+    def AddSlice(self, arr):
+        if arr.dtype == 'uint16':
+            nptype = 'uint16'
+            piltype = 'I;16'
+        #elif arr.dtype == 'uint8':
+        #    nptype = 'uint8'
+        #    piltype = 'I;8'
+        else:
+            nptype = 'f'
+            piltype = 'F' 
+        
+        ii = Image.fromarray(arr.astype(nptype), piltype)
+
+        self.fp.seek(0,2) # go to end of file
+        if self.z==0:
+            # ref. PIL  TiffImagePlugin
+            # PIL always starts the first IFD at offset 8
+            ifdOffset = 8
+        else:
+            ifdOffset = self.fp.tell()
+
+        ii.save(self.fp, format="TIFF", **self.params)
+
+        if self.z>0: # correct "next" entry of previous ifd -- connect !
+            ifdo = self.ifd_offsets[-1]
+            self.fp.seek(ifdo)
+            ifdLength = ii._debug_multipage.i16(self.fp.read(2))
+            self.fp.seek(ifdLength*12,1) # go to "next" field near end of ifd
+            self.fp.write(ii._debug_multipage.o32( ifdOffset ))
+
+        self.ifd_offsets.append(ifdOffset)
+        self.z += 1
+    
+    def close(self):
+        self.fp.close()
+    

PYME/IO/FileUtils/setup.py

@@ -0,0 +1,36 @@
+#!/usr/bin/python
+
+##################
+# setup.py
+#
+# Copyright David Baddeley, 2009
+# d.baddeley@auckland.ac.nz
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+
+
+def configuration(parent_package='',top_path=None):
+    from numpy.distutils.misc_util import Configuration
+    config = Configuration('FileUtils',parent_package,top_path)
+    #config.add_subpackage('abf')
+    
+    #config.make_svn_version_py()  # installs __svn_version__.py
+    #config.make_config_py()
+    return config
+
+if __name__ == '__main__':
+    from numpy.distutils.core import setup
+    setup(**configuration(top_path='').todict())

PYME/IO/FileUtils/sf-thumbnailer.py

@@ -0,0 +1,76 @@
+#!/usr/bin/python
+
+##################
+# sf-thumbnailer.py
+#
+# Copyright David Baddeley, 2009
+# d.baddeley@auckland.ac.nz
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+
+#!/usr/bin/python
+
+#import logging
+#LOG_FILENAME = '/tmp/sf-thumbnailer.log'
+#logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG,)
+
+import sys
+import gnomevfs
+
+try:
+    # noinspection PyCompatibility
+    import cPickle
+except ImportError:
+    #py3
+    import pickle as cPickle
+
+if __name__ == '__main__':
+    import matplotlib
+    matplotlib.use('Agg')
+    
+    # from pylab import *
+    from matplotlib.pyplot import *
+    from numpy import *
+    
+    dpi = 100.
+    
+    
+    inputFile = gnomevfs.get_local_path_from_uri(sys.argv[1])
+    outputFile = sys.argv[2]
+    thumbSize = int(sys.argv[3])
+    
+    #logging.debug('Input File: %s\n' % inputFile)
+    #logging.debug('Ouput File: %s\n' % outputFile)
+    #logging.debug('Thumb Size: %s\n' % thumbSize)
+    
+    #def generateThumbnail(inputFile, thumbsize):
+    fid = open(inputFile)
+    spx, spy = cPickle.load(fid)
+    fid.close()
+    
+    f = figure(figsize=(thumbSize/dpi, 0.5*thumbSize/dpi))
+    
+    axes([0, 0, 1, 1])
+    xin, yin = meshgrid(arange(0, 512*70, 4000), arange(0, 256*70, 4000))
+    xin = xin.ravel()
+    yin = yin.ravel()
+    quiver(xin, yin, spx.ev(xin, yin), spy.ev(xin, yin), scale=2e3)
+    xticks([])
+    yticks([])
+    axis('image')
+    
+    f.savefig(outputFile, dpi=dpi, format='png')
+

PYME/IO/FileUtils/stl.py

@@ -0,0 +1,44 @@
+import numpy as np
+
+
+def load_stl_binary(fn):
+    """
+    Load triangles from binary STL file.
+
+    Parameters
+    ----------
+    fn : string
+        File name to read in
+
+    Returns
+    -------
+    A set of triangles.
+    """
+    with open(fn, 'rb') as f:
+        f.seek(80)
+        n_triangles = np.fromfile(f, 'int32', 1).squeeze()
+        triangles = np.fromfile(f, [('normal', '3f4'), ('vertex0', '3f4'), ('vertex1', '3f4'), ('vertex2', '3f4'), ('attrib', 'u2')], n_triangles)
+
+        return triangles
+
+
+def save_stl_binary(fn, data):
+    """
+    Save list of triangles to binary STL file.
+
+    Parameters
+    ----------
+    fn : string
+        File name to write
+    data : np.array
+        A list of triangles in the same format as returned by load_stl_binary
+
+    Returns
+    -------
+    None
+    """
+
+    with open(fn, 'wb') as f:
+        f.write(np.zeros(80,dtype='uint8'))
+        f.write(np.uint32(data.shape[0]))
+        data.tofile(f)

PYME/IO/FileUtils/thumbnailDatabase.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python
+
+##################
+# thumbnailDatabase.py
+#
+# Copyright David Baddeley, 2009
+# d.baddeley@auckland.ac.nz
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+
+import sqlite3
+try:
+    import cPickle as pickle
+except ImportError:
+    #py3
+    import pickle
+    
+import os
+import zlib
+from numpy import ndarray
+
+THUMBSIZE=200
+thumbnailers = {'.h5':'h5-thumbnailer', '.h5r':'h5r-thumbnailer', '.kdf':'kdf-thumbnailer'}
+
+#teach sqlite about numpy arrays
+def adapt_numarray(array):
+    return sqlite3.Binary(zlib.compress(array.dumps()))
+
+def convert_numarray(s):
+    return pickle.loads(zlib.decompress(s))
+
+sqlite3.register_adapter(ndarray, adapt_numarray)
+sqlite3.register_converter("ndarray", convert_numarray)
+
+dbOpen = False
+thumbDB = None
+
+def openDB():
+    global dbOpen, thumbDB
+    
+    if not dbOpen:
+        dbOpen = True
+        thumbDB = sqlite3.connect('/srv/PYME/PYMEThumbnails.db', detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
+        thumbDB.isolation_level = None
+        
+        #see what tables we've got defined
+        tableNames = [a[0] for a in thumbDB.execute('SELECT name FROM sqlite_master WHERE type="table"').fetchall()]
+        
+        #if we haven't already got a thumbnail database, create one
+        if not 'Thumbnails' in tableNames:
+            thumbDB.execute("CREATE TABLE Thumbnails (filename text, thumbnail ndarray)")
+
+
+
+
+def getThumbnail(filename):
+    openDB()
+    #strip extra leading / (inserted for some reason by test server)
+    if filename.startswith('//'):
+        filename = filename[1:]
+
+    #if missing leading /, add
+    if not filename.startswith('/'):
+        filename = '/' + filename
+
+    ret = thumbDB.execute("SELECT thumbnail FROM Thumbnails WHERE filename=?", (filename,)).fetchone()
+
+    if ret is None:
+        #cludge around old entries with extra /
+        ret = thumbDB.execute("SELECT thumbnail FROM Thumbnails WHERE filename=?", ('/' + filename,)).fetchone()
+
+
+    if ret is None:
+        ext = os.path.splitext(filename)[-1]
+
+        if ext in thumbnailers.keys():
+            #print ext
+            #try:
+            #thumbMod = __import__(thumbnailers[ext])
+            thumbMod = __import__('PYME.IO.FileUtils.' + thumbnailers[ext], fromlist=['PYME', 'io', 'FileUtils'])
+
+            ret = thumbMod.generateThumbnail(filename, THUMBSIZE)
+
+            thumbDB.execute("INSERT INTO Thumbnails VALUES (?, ?)", (filename, ret))
+            thumbDB.commit()
+            #except:
+            #    pass
+    else:
+        ret = ret[0]
+
+    return ret

PYME/IO/HTTPSpooler_v2.py

@@ -0,0 +1,141 @@
+#!/usr/bin/python
+
+##################
+# QueueSpooler.py
+#
+# Copyright David Baddeley, 2009
+# d.baddeley@auckland.ac.nz
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+
+#from PYME.IO import MetaDataHandler
+from PYME.IO.events import MemoryEventLogger
+import PYME.IO.Spooler as sp
+
+import os
+# import time
+# import threading
+
+# try:
+#     # noinspection PyCompatibility
+#     import Queue # type: ignore
+# except ImportError:
+#     #py3
+#     import queue as Queue
+
+from PYME.IO import clusterIO
+#from PYME.IO import cluster_streaming
+#from PYME.IO import PZFFormat
+from PYME import config
+
+#import random
+import logging
+logger = logging.getLogger(__name__)
+
+
+def getReducedFilename(filename):
+    #rname = filename[len(nameUtils.datadir):]
+        
+    sname = '/'.join(filename.split(os.path.sep))
+    if sname.startswith('/'):
+        sname = sname[1:]
+    
+    return sname
+    
+def exists(seriesName):
+    return clusterIO.exists(getReducedFilename(seriesName))
+
+
+from PYME.IO import acquisition_backends
+
+class Spooler(sp.Spooler):
+    def __init__(self, filename, frameSource, frameShape, **kwargs):
+        sp.Spooler.__init__(self, filename, frameSource, **kwargs)
+        
+        self.seriesName = getReducedFilename(filename)        
+        self._aggregate_h5 = kwargs.get('aggregate_h5', False)
+        
+        self.clusterFilter = kwargs.get('serverfilter', 
+                                        config.get('dataserver-filter', ''))
+        
+        
+        chunk_size = config.get('httpspooler-chunksize', 50)
+        def dist_fcn(n_servers, i=None):
+            if i is None:
+                # distribute at random
+                import random
+                return random.randrange(n_servers)
+            
+            return int(i/chunk_size) % n_servers
+        
+        
+        self._backend = acquisition_backends.ClusterBackend(self.seriesName, 
+                                                            distribution_fcn=dist_fcn, 
+                                                            compression_settings=kwargs.get('compressionSettings', {}),
+                                                            cluster_h5=self._aggregate_h5,
+                                                            serverfilter=self.clusterFilter,
+                                                            shape=[-1,-1,1,-1,1], #spooled aquisitions are time series (for now)
+                                                            )
+        
+        self.md = self._backend.mdh
+        self.evtLogger = MemoryEventLogger(self, time_fcn=self._time_fcn)
+        self._stopping = False
+                
+    def finished(self):
+        # FIXME - this probably needs a bit more work.
+        return self._stopping
+        
+    def getURL(self):
+        #print CLUSTERID, self.seriesName
+        return 'PYME-CLUSTER://%s/%s' % (self.clusterFilter, self.seriesName)
+        
+    def StartSpool(self):
+        sp.Spooler.StartSpool(self)
+        
+        logger.debug('Starting spooling: %s' %self.seriesName)
+
+        self._backend.initialise()
+    
+    def finalise(self):
+        # wait until our input queue is empty rather than immediately stopping saving.
+        self._stopping=True
+        logger.debug('Stopping spooling %s' % self.seriesName)
+        
+        #TODO - add option for non-blocking finalise??
+        self._backend.finalise(events=None)
+        
+        # save events
+        # do this here, **after** we have joined all the pushing threads to make sure that writing to all nodes has
+        # finished when we save events.
+        # TODO - use a binary format for saving events - they can be quite
+        # numerous, and can trip the standard 1 s clusterIO.put_file timeout.
+        clusterIO.put_file(self._backend._series_location + '/events.json', 
+                               self.evtLogger.to_JSON().encode(),
+                               self.clusterFilter, timeout=10)
+        
+
+    def OnFrame(self, sender, frameData, **kwargs):
+        self._backend.store_frame(self.imNum, frameData)
+
+        sp.Spooler.OnFrame(self)
+        
+    def cleanup(self):
+        del self._backend
+     
+
+
+
+