python-microscopy 25.11.20__cp313-cp313-win_amd64.whl

PYME/IO/DataSources/BufferedDataSource.py

@@ -0,0 +1,99 @@
+#!/usr/bin/python
+##################
+# BufferedDataSource.py
+#
+# Copyright David Baddeley, 2011
+# d.baddeley@auckland.ac.nz
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+import numpy
+from .BaseDataSource import XYZTCDataSource, XYZTCWrapper
+import threading
+
+class DataSource(XYZTCDataSource): #buffer our io to avoid decompressing multiple times
+    moduleName = 'BufferedDataSource'
+    def __init__(self,datasource, bLen = 12):
+        
+        if (not isinstance(datasource, XYZTCDataSource)) and (not datasource.ndim == 5) :
+            datasource = XYZTCWrapper.auto_promote(datasource)
+        
+        self.bLen = bLen
+        self.buffer = None #delay creation until we know the dtype
+        #self.buffer = numpy.zeros((bLen,) + dataSource.getSliceShape(), 'uint16')
+        self.insertAt = 0
+        self.bufferedSlices = -1*numpy.ones((bLen,), 'i')
+        self.dataSource = datasource
+
+        self.lock = threading.Lock()
+
+        size_z, size_t, size_c = datasource.shape[2:]
+        XYZTCDataSource.__init__(self, input_order=datasource._input_order, size_z=size_z, size_t=size_t, size_c=size_c)
+
+    def getSlice(self,ind):
+        #global bufferMisses
+        #print self.bufferedSlices, self.insertAt, ind
+        #return self.dataSource.getSlice(ind)
+        with self.lock:
+            #print ind
+            if ind in self.bufferedSlices: #return from buffer
+                #print int(numpy.where(self.bufferedSlices == ind)[0])
+                #print self.bufferedSlices
+                ret = self.buffer[int(numpy.where(self.bufferedSlices == ind)[0]), :, :].copy()
+                #print 'buf'
+            else: #get from our data source and store in buffer
+                sl = self.dataSource.getSlice(ind)
+                self.bufferedSlices[self.insertAt] = ind
+                #print sl.shape
+                #print self.insertAt
+                #print self.buffer
+
+                if self.buffer is None: #buffer doesn't exist yet
+                    self.buffer = numpy.zeros((self.bLen, ) + self.dataSource.getSliceShape(), sl.dtype)
+
+                #print self.buffer.shape
+
+                self.buffer[self.insertAt,:, :] = sl
+                self.insertAt += 1
+                self.insertAt %= self.bLen
+
+                #bufferMisses += 1
+
+                ret = sl
+            return ret
+    
+    def getSliceShape(self):
+        #return (self.im.size[1], self.im.size[0])
+        return self.dataSource.getSliceShape()
+        #return self.data.shape[:2]
+
+    def getNumSlices(self):
+        return self.dataSource.getNumSlices()
+
+    def getEvents(self):
+        return self.dataSource.getEvents()
+
+    def release(self):
+        return self.dataSource.release()
+
+    def reloadData(self):
+        return self.dataSource.reloadData()
+
+    @property
+    def is_complete(self):
+        return self.dataSource.is_complete()
+        
+
+

PYME/IO/DataSources/ClusterPZFDataSource.py

@@ -0,0 +1,124 @@
+#!/usr/bin/python
+
+##################
+# HDFDataSource.py
+#
+# Copyright David Baddeley, 2009
+# d.baddeley@auckland.ac.nz
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+
+#from PYME.ParallelTasks.relativeFiles import getFullFilename
+#import tables
+from .BaseDataSource import XYZTCDataSource
+
+#import httplib
+#import urllib
+#import requests
+#import cPickle as pickle
+import time
+import json
+#import pandas as pd
+import numpy as np
+SHAPE_LIFESPAN = 5
+
+from PYME.IO import clusterIO
+from PYME.IO import PZFFormat
+from PYME.IO import MetaDataHandler
+
+class DataSource(XYZTCDataSource):
+    moduleName = 'ClusterPZFDataSource'
+    def __init__(self, url, queue=None):
+        self.seriesName = url
+        #print url
+        self.clusterfilter = url.split('://')[1].split('/')[0]
+        #print self.clusterfilter
+        self.sequenceName = url.split('://%s/' % self.clusterfilter)[1]
+        #print self.sequenceName
+        self.lastShapeTime = 0
+        
+        mdfn = '/'.join([self.sequenceName, 'metadata.json'])  
+        
+        #print mdfn
+        
+        self.mdh = MetaDataHandler.NestedClassMDHandler()
+        self.mdh.update(json.loads(clusterIO.get_file(mdfn, self.clusterfilter)))
+        
+        self.fshape = None#(self.mdh['Camera.ROIWidth'],self.mdh['Camera.ROIHeight'])
+        
+        self._getNumFrames()
+
+        dimorder= self.mdh.get('DimOrder', 'XYZTC')
+        size_z = self.mdh.get('SizeZ', -1)
+        size_c = self.mdh.get('SizeC', 1)
+        size_t = self.mdh.get('SizeT', 1)
+
+        # if the series is complete when we start, we don't need to update the number of slices
+        self._complete = clusterIO.exists(self.eventFileName, self.clusterfilter)
+        
+        XYZTCDataSource.__init__(self, dimorder, size_z=size_z, size_t=size_t, size_c=size_c)
+    
+    def _getNumFrames(self):
+        frameNames = [f for f in clusterIO.listdir(self.sequenceName, self.clusterfilter) if f.endswith('.pzf')]
+        self.numFrames = len(frameNames)
+        self.lastShapeTime = time.time()
+    
+    def getSlice(self, ind):
+        frameName = '%s/frame%05d.pzf' % (self.sequenceName, ind)
+        sl = PZFFormat.loads(clusterIO.get_file(frameName, self.clusterfilter))[0]
+        
+        #print sl.shape, sl.dtype
+        return sl.squeeze()
+
+    def getSliceShape(self):
+        if self.fshape is None:
+            self.fshape = self.getSlice(0).shape
+        return self.fshape
+        
+    def getNumSlices(self):
+        if not self._complete:
+            t = time.time()
+            if (t-self.lastShapeTime) > SHAPE_LIFESPAN:
+                self._getNumFrames()
+            
+        return self.numFrames
+
+    @property
+    def eventFileName(self):
+        return self.sequenceName + '/events.json'
+
+    def getEvents(self):
+        from PYME.IO import events
+        import json
+        try:
+            ev = json.loads(clusterIO.get_file(self.eventFileName, self.clusterfilter, timeout=10))
+            return events.EventLogger.list_to_array(ev)
+        except (IOError, ValueError):
+            #our series might not have any events
+            return []
+        
+    def getMetadata(self):
+        return self.mdh
+
+    @property
+    def is_complete(self):
+        if not self._complete:
+            # if cached property is false, check to see if anything has changed
+            #TODO - add check to see if we have an updated number of frames
+            self._complete = clusterIO.exists(self.eventFileName, self.clusterfilter)
+        
+        return self._complete
+ 

PYME/IO/DataSources/ConcatenatedDataSource.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python
+
+"""
+@author: zacsimile
+"""
+import numpy as np
+from numpy.core.fromnumeric import searchsorted
+from .BaseDataSource import XYZTCDataSource
+
+class DataSource(XYZTCDataSource):
+    """
+    This lets us concatenate datasources and access them via a single data source.
+    """
+    def __init__(self, datasources, *args, **kwargs):
+
+
+        self.n_datasources = len(datasources)
+        self.datasources = datasources
+        self._slice_shape = self.datasources[0].getSliceShape()
+
+        for ds in self.datasources[1:]:
+            assert(ds.getSliceShape() == self._slice_shape)
+
+        self.datasource_index = np.hstack([0, np.cumsum([ds.getNumSlices() for ds in self.datasources])])
+        self._n_slices = self.datasource_index[-1]
+
+        if not (kwargs.get('size_z', None) and kwargs.get('size_t', None) and kwargs.get('size_c', None)):
+            # should also conveniently work for 0,0,0
+            kwargs['size_z'] = self._n_slices
+     
+        XYZTCDataSource.__init__(self, *args, **kwargs)
+
+    def getSlice(self, ind):
+        i = np.searchsorted(self.datasource_index, ind, side='right')-1
+        rem = ind-self.datasource_index[i]
+        return self.datasources[i].getSlice(rem)
+       
+    def getNumSlices(self):
+        return self._n_slices
+
+    def getSliceShape(self):
+        return self._slice_shape
+
+    def getEvents(self):
+        events = []
+        for ds in self.datasources:
+            events.append(ds.getEvents())
+        return np.hstack(events)
+
+    def release(self):
+        for ds in self.datasources:
+            ds.release()
+
+    def reloadData(self):
+        for ds in self.datasources:
+            ds.reloadData()

PYME/IO/DataSources/CropDataSource.py

@@ -0,0 +1,243 @@
+#!/usr/bin/python
+##################
+# UnsplitDataSource.py
+#
+# Copyright David Baddeley, 2011
+# d.baddeley@auckland.ac.nz
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+import numpy as np
+from scipy import ndimage
+from .BaseDataSource import XYTCDataSource, XYTCWrapper, XYZTCDataSource, BaseDataSource, XYZTCWrapper
+
+class _DataSource(XYTCDataSource):
+    moduleName = 'CropDataSource'
+    def __init__(self,dataSource, xrange=None, yrange=None, trange=None):
+        #self.unmixer = unmixer
+        self.dataSource = dataSource
+        
+        if xrange is None:
+            self.xrange = (0, self.dataSource.shape[0])
+        else:
+            self.xrange = xrange
+            
+        if yrange is None:
+            self.yrange = (0, self.dataSource.shape[1])
+        else:
+            self.yrange = yrange
+            
+        if trange is None:
+            self.trange = (0, self.dataSource.getNumSlices())
+        else:
+            self.trange = trange
+        
+    
+    def getSlice(self,ind):
+        sl = self.dataSource.getSlice(ind + self.trange[0])
+        
+        return sl[self.xrange[0]:self.xrange[1], self.yrange[0]:self.yrange[1]]
+
+    def getSliceShape(self):
+        #return (self.im.size[1], self.im.size[0])
+        return (self.xrange[1] - self.xrange[0], self.yrange[1] - self.yrange[0])
+        #return self.data.shape[:2]
+
+    def getNumSlices(self):
+        return self.trange[1] - self.trange[0]
+
+    def getEvents(self):
+        return self.dataSource.getEvents()
+
+    def release(self):
+        return self.dataSource.release()
+        
+
+    def reloadData(self):
+        return self.dataSource.reloadData()
+
+class DataSource(XYZTCDataSource):
+    moduleName = 'CropDataSource'
+
+    @classmethod
+    def _sliceify(cls, range, nmax):
+        if range is None:
+            return slice(0, nmax, 1)
+        else:
+            if not isinstance(range, slice):
+                range = slice(*range)
+            
+            return slice(*range.indices(nmax))
+
+    def __init__(self,dataSource, xrange=None, yrange=None, zrange=None, trange=None):
+        #self.unmixer = unmixer
+        self.dataSource = dataSource #type: XYZTCDataSource
+
+        # check than input datasource is xyztc (duck-typed by checking dimension number so that 5D ListWraps are also OK) 
+        assert(isinstance(dataSource, BaseDataSource))
+        assert(dataSource.ndim == 5)
+
+        if not isinstance(dataSource, XYZTCDataSource):
+            # if not an XYZTCDataSource, wrap as such (needed to get stride info etc ...)
+            # this should trigger on 5D ListWraps and ArrayDataSources
+            self.dataSource = XYZTCWrapper.auto_promote(dataSource)
+
+        self.xslice = self._sliceify(xrange, self.dataSource.shape[0])
+        self.yslice = self._sliceify(yrange, self.dataSource.shape[1])
+        self.zslice = self._sliceify(zrange, self.dataSource.shape[2])
+        self.tslice = self._sliceify(trange, self.dataSource.shape[3])
+        
+        
+        szs = [int(np.floor((r.stop-r.start)/r.step)) for r in [self.xslice, self.yslice, self.zslice, self.tslice]] + [self.dataSource.shape[4],]
+
+        self.set_dim_order_and_size(self.dataSource._input_order, szs[2], szs[3], szs[4])
+        
+        self._shape = tuple(szs)
+        self._dtype = dataSource.dtype
+        
+        self._i_z_stride = self.dataSource._z_stride
+        self._i_t_stride = self.dataSource._t_stride
+        self._i_c_stride = self.dataSource._c_stride
+
+
+    
+    def getSlice(self,ind):
+        o_strides = np.array((self._z_stride, self._t_stride, self._c_stride))
+        i_strides = np.array((self._i_z_stride*self.zslice.step, self._i_t_stride*self.tslice.step, self._i_c_stride)).astype('i')
+        i_offsets = np.array((self.zslice.start, self.tslice.start, 0))
+
+        stride_order = np.argsort(o_strides)
+        #print(stride_order, o_strides, i_strides)
+        so_strides = o_strides[stride_order]
+
+        i = int(np.floor(ind/so_strides[-1]))
+        j = int(np.floor((ind-i*so_strides[-1])/so_strides[-2]))
+        k = ind - i*so_strides[-1] -j*so_strides[-2]
+
+        ijk = int(((np.array([k,j,i]) + i_offsets[stride_order])*i_strides[stride_order]).sum())
+
+        #print(ind, ijk, (k, j, i))
+
+        sl = self.dataSource.getSlice(ijk)
+
+        #print(sl.shape)
+        
+        sl = sl[self.xslice, self.yslice]
+
+        #print(sl.shape)
+        return sl
+
+    def getSliceShape(self):
+        #return (self.im.size[1], self.im.size[0])
+        #return (self.xrange[1] - self.xrange[0], self.yrange[1] - self.yrange[0])
+        return self._shape[:2]
+
+    def getNumSlices(self):
+        return np.prod(self.shape[2:])
+
+    def getEvents(self):
+        return self.dataSource.getEvents()
+
+    def release(self):
+        return self.dataSource.release()  
+
+    def reloadData(self):
+        return self.dataSource.reloadData()
+        
+
+
+def crop_image(image, xrange=None, yrange=None, zrange=None, trange=None):
+    from PYME.IO.image import ImageStack
+    from PYME.IO.DataSources import ArrayDataSource
+
+    vx, vy, vz = image.voxelsize
+    ox, oy, oz = image.origin
+
+    def _offset(r):     
+        if r is None:
+            return 0
+        elif isinstance(r, slice):
+            return r.start
+        else:
+            return r[0]
+        
+    def _slice(r):
+        if isinstance(r, slice):
+            return r
+        else: 
+            # assume tuple or list
+            # will raise TypeError if None
+            return slice(*r)
+
+    if isinstance(image.data_xyztc, ArrayDataSource.XYZTCArrayDataSource):
+        # special case for array data source so that we can use views and get optimal chunk access (dask/zarr)
+        #  - use array slicing rather slice slicing
+        # TODO - change CropDataSource to be chunk aware
+        s = image.data_xyztc.shape
+        cp_a = image.data_xyztc[DataSource._sliceify(xrange, s[0]), DataSource._sliceify(yrange, s[1]), DataSource._sliceify(zrange, s[2]), DataSource._sliceify(trange, s[3]), :]
+        cropped = ArrayDataSource.XYZTCArrayDataSource(cp_a)
+    else:
+        cropped = DataSource(image.data_xyztc, xrange=xrange, yrange=yrange, zrange=zrange, trange=trange)
+    
+    im = ImageStack(cropped, titleStub='Cropped Image')
+    im.mdh.copyEntriesFrom(image.mdh)
+    im.mdh['Parent'] = image.filename
+    #im.mdh['Processing.CropROI'] = roi
+    
+    im.mdh['Origin.x'] = ox + vx*_offset(xrange)
+    im.mdh['Origin.y'] = oy + vy*_offset(yrange)
+    im.mdh['Origin.z'] = oz + vz*_offset(zrange)
+
+    try:
+        im.mdh['cropping.xslice'] = _slice(xrange).indices(image.data_xyztc.shape[0])
+    except TypeError:
+        #xrange is None
+        pass
+
+    try:
+        im.mdh['cropping.yslice'] = _slice(yrange).indices(image.data_xyztc.shape[1])
+    except TypeError:
+        #yrange is None
+        pass
+
+    try:
+        im.mdh['cropping.zslice'] = _slice(zrange).indices(image.data_xyztc.shape[2])
+    except TypeError:
+        #zrange is None
+        pass
+
+    try:
+        im.mdh['cropping.tslice'] = _slice(trange).indices(image.data_xyztc.shape[3])
+    except TypeError:
+        #trange is None
+        pass
+
+    return im
+
+def roi_crop_image(image, roi, z=True, t=False):
+    if z:
+        zrange = roi[2]
+    else:
+        zrange = None
+
+    if t:
+        trange = roi[3]
+    else:
+        trange = None
+
+    xrange = roi[0]
+    yrange = roi[1]
+
+    return crop_image(image, xrange, yrange, zrange, trange)

PYME/IO/DataSources/DcimgDataSource.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+##################
+# HDFDataSource.py
+#
+# Copyright David Baddeley, 2009
+# d.baddeley@auckland.ac.nz
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+
+from PYME.IO.FileUtils.nameUtils import getFullExistingFilename
+from .BaseDataSource import XYTCDataSource
+
+from PYME.IO import dcimg
+
+class DataSource(XYTCDataSource):
+    moduleName = 'DcimgDataSource'
+    def __init__(self, filename, taskQueue=None):
+        self.filename = getFullExistingFilename(filename)#convert relative path to full path
+        
+        self.DCIFile = dcimg.DCIMGFile(self.filename)
+        
+    def getSlice(self, ind):
+        #print ind
+        return self.DCIFile.get_frame(ind)
+
+    def getSliceShape(self):
+        return self.DCIFile.get_slice_shape()
+
+    def getNumSlices(self):
+        return self.DCIFile.get_num_slices()
+
+    def getEvents(self):
+        return []
+
+    def release(self):
+        del self.DCIFile
+
+    def reloadData(self):
+        self.release()
+        self.DCIFile = dcimg.DCIMGFile(self.filename)

PYME/IO/DataSources/ElectronsPerSecondDataSource.py

@@ -0,0 +1,61 @@
+#!/usr/bin/python
+
+##################
+# ElectronsPerSecondDataSource.py
+#
+# Copyright Andrew Barentine, David Baddeley, 2022
+# d.baddeley@auckland.ac.nz
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##################
+
+from PYME.IO.DataSources.FlatFieldDataSource import DataSource as FlatFieldDataSource
+
+
+class DataSource(FlatFieldDataSource):
+    moduleName = 'ElectronsPerSecondDataSource'
+    def __init__(self, parentSource, mdh, flatfield=None, dark=None):
+        """
+        Create a datasource which wraps a series in units of ADU and returns frames in units of
+        photoelectrons per second (e/s). parentSource should not be camera map corrected (dark-
+        or flatfield-corrected).
+
+        Warnings
+        --------
+
+        A lot of PYME assumes that image units are in ADUs. This means that it would be easy to end up doing this correction
+        in downstream modules as well as here and ending up with erroneous data as a result. Safe usage of this module in
+        complex workflows will require building enhanced unit awareness into other parts of PYME. 
+        
+        This is discussed in more detail in 
+        PYME.recipes.processing.RawADUToElectronsPerCount
+        
+        """
+        if mdh.getOrDefault('Units.Intensity', 'ADU') == 'e/s':
+            # FIXME - the name of this key might change (maybe Units.Intensity or similar)  
+            # TODO - rather than hard throw just don't further correct the parentSource
+            raise RuntimeError('units are already e/s')
+        
+        FlatFieldDataSource.__init__(self, parentSource, mdh, flatfield, dark)
+        
+        self._adu_to_epers = self.mdh['Camera.ElectronsPerCount'] / self.mdh['Camera.TrueEMGain'] / self.mdh['Camera.IntegrationTime']
+
+        # set a units property on the datasource
+        # this might not end up getting used, but is a nice failsafe for situations where we don't propagate metadata
+        self.units='e/s'
+
+    def getSlice(self, ind):
+        # flatfield getSlice will subtract the dark map and flatfield. 
+        return FlatFieldDataSource.getSlice(self, ind) * self._adu_to_epers  # corrected ADU to photoelectrons/second