python-microscopy 25.6.5__cp39-cp39-win_amd64.whl

PYME/experimental/aqavg.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Sep 12 17:00:16 2014
+
+@author: David Baddeley
+"""
+
+import numpy as np
+
+class FrameAverager(object):
+    def __init__(self, pa, nAvg=10):
+        self.nAvg = nAvg
+        self.pa = pa
+        
+        self.nCurrent = 0        
+        self.frameBuff = []
+        self.frameSum = np.zeros(pa.dsa.shape, 'float64')
+        self.frameAvg = np.zeros(pa.dsa.shape, 'float64')
+        
+        self.pa.WantFrameNotification.append(self.tick)
+        
+    def tick(self, caller=None):
+        if not self.frameSum.shape == self.pa.dsa.shape:
+            self.frameSum = np.zeros(self.pa.dsa.shape, 'float64')
+            self.frameAvg = np.zeros(self.pa.dsa.shape, 'float64')
+            self.nCurrent = 0
+            self.frameBuff.clear()
+        
+        if self.nCurrent >= self.nAvg:
+            ff = self.frameBuff.pop(0)
+            self.frameSum -= ff
+            self.nCurrent -= 1
+            
+        self.frameBuff.append(1.0*self.pa.dsa)
+        self.frameSum += self.frameBuff[-1]
+        self.nCurrent += 1
+        
+        self.frameAvg[:] = (self.frameSum/self.nCurrent)[:]
+        
+    

PYME/experimental/dcimgFileChucker.py

@@ -0,0 +1,375 @@
+
+import sys
+import os
+import errno
+import glob
+import time
+import datetime
+import traceback
+
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+logger = logging.getLogger(__name__)
+
+import PYME.experimental.dcimgSpoolShim as DCIMGSpool
+from PYME.IO import PZFFormat
+
+def safe_remove(path):
+    """
+    File removal with error handling. Defining function to clean up the FileChucker code
+    
+    Parameters
+       ----------
+       path : str
+           The fully resolved filename of the file to be deleted
+       Returns
+       -------
+       
+    """
+    try:
+        os.remove(path)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+
+class DumpsterRaccoon(object):
+    """
+
+    The DumpsterRaccoon does keeps track of files we try to delete before a non-PYME acquisition software releases the
+    file. The raccoon has methods to retry deletion, but is currently not run in its own thread, and therefor these
+    methods are called explicitly by the FileChucker.
+
+    """
+    def __init__(self, num_retries=3):
+        """
+
+        Parameters
+        ----------
+        num_retries : int
+            Maximum number of attempts to be made in deleting any given file before ignoring it
+
+        """
+        self.retries = num_retries
+        self.dumpster = {}
+
+    def __len__(self):
+        """
+
+        Returns
+        -------
+        num_paths : int
+            The number of files which are currently queued to be removed
+
+        """
+        return len(self.dumpster.keys())
+
+    def hoard_morsel(self, path):
+        """
+
+        Increments the number of attempts that have been made to remove a file and handles deletion of that file from
+        the queue if self.retries is exceeded.
+
+        Parameters
+        ----------
+        path : str
+            Full file path to the file to be garbage-collected
+
+        Returns
+        -------
+
+        """
+        try:  # if Raccoon already knows about the trash, increment the number of times he's tried to eat it
+            self.dumpster[path] += 1
+            if self.dumpster[path] < self.retries:
+                self.dumpster[path] += 1
+            else:  # this one's just not going down, forget about it
+                del self.dumpster[path]
+
+        except KeyError:  # this is the first time Raccoon has seen this trash, store for later consumption
+            self.dumpster[path] = 0
+
+    def eat_trash(self, path):
+        """
+
+        Try to delete a single file. If it fails, hoard the tasty morsel for later consumption.
+
+        Parameters
+        ----------
+        path : str
+            Full file path to the file to be garbage-collected
+
+        Returns
+        -------
+
+        """
+        try:
+            safe_remove(path)
+            # if we're successful, make sure to remove this morsel from our memory
+            self.dumpster.pop(path, None)
+        except Exception:
+            self.hoard_morsel(path)
+            logger.exception(traceback.format_exc())
+
+    def skim_trash(self):
+        """
+
+        Try to remove a single piece of trash from the queue.
+
+        Returns
+        -------
+
+        """
+        if len(self) > 0:
+            self.eat_trash(self.dumpster.keys()[0])
+
+    def dumpster_fire(self):
+        """
+
+        Try to remove each piece of trash currently in the queue.
+
+        Returns
+        -------
+
+        """
+        for k in self.dumpster.keys():
+            self.eat_trash(k)
+
+
+
+class venerableFileChucker(object):
+    """
+    fileChucker searches a given folder and hucks the DCIMG files it finds there onto the cluster.
+    Once started, it does not stop unless slain by the user.
+    This is certainly not the most elegant way of implementing the DCIMGSpooler, but may suffice for now....
+
+    """
+    def __init__(self, searchFolder, timeout=12, quantize=False):
+        """
+
+        Parameters
+        ----------
+        searchFolder : str
+            Path the the folder to watch
+        timeout : float
+            time after which a series is deemed to be dead, in s. Default is 1 hr.
+        """
+        self.folder = searchFolder
+
+        self.spooler = DCIMGSpool.DCIMGSpoolShim(timeout=timeout)
+        self.timeout = timeout
+        self.comp_settings = {'quantization': PZFFormat.DATA_QUANT_SQRT if quantize else PZFFormat.DATA_QUANT_NONE}
+
+        self.dumpster_raccoon = DumpsterRaccoon()
+
+
+    def _spoolSeries(self, mdfilename, delete_after_spool=False):
+        """
+        Spool a single series, which already exists
+
+        Parameters
+        ----------
+        mdfilename : str
+            The fully resolved filename of the series metadata
+
+        Returns
+        -------
+
+        """
+        self.spooler.OnNewSeries(mdfilename, self.comp_settings)
+        series_stub =  mdfilename.strip('.json')
+        print(datetime.datetime.utcnow())
+
+        # find chunks for this metadata file (this should return the full paths)
+        chunkList = sorted(glob.glob(series_stub + '*.dcimg'))
+
+        for chunk in chunkList:
+            try:
+                self.spooler.OnDCIMGChunkDetected(chunk)
+                #time.sleep(.5)   # FIXME: When using HUFFMANCODE more frames are lost without waiting
+                #wait until we've sent everything
+                #this is a bit of a hack
+                #time.sleep(.1)
+                while not self.spooler.spooler.finished():
+                    time.sleep(.1)
+            finally:
+                if delete_after_spool:
+                    self.dumpster_raccoon.eat_trash(chunk)
+
+        # spool _events.json
+        events_filename = series_stub + '_events.json' #note that we ignore this at present, kept for future compatibility
+        zsteps_filename = series_stub + '_zsteps.json'
+
+        try:
+            self.spooler.OnSeriesComplete(events_filename, zsteps_filename, pushTaskToCluster=True)
+
+            print(datetime.datetime.utcnow())
+            # TODO: Add Feedback from cluster and also speed up writing in cluster
+            # time.sleep(10)
+        finally:
+            if delete_after_spool:
+                self.dumpster_raccoon.dumpster_fire()
+                self.dumpster_raccoon.eat_trash(mdfilename)
+                self.dumpster_raccoon.eat_trash(events_filename)
+                self.dumpster_raccoon.eat_trash(zsteps_filename)
+
+    def huck(self, mdFile, delete_after_spool, only_spool_complete):
+        """
+        Spools a series to the cluster
+        Parameters
+        ----------
+        mdFile : str
+            Fully resolved path to metadata file for series to be spooled
+        delete_after_spool : bool
+            Flag to determine whether files are deleted after spooling (True) or spooled without deletion (False)
+        only_spool_complete : bool
+            Flag to determine whether dcimg files should be spooled as soon as they are written (False), i.e. before an
+            events file is written, or whether the FileChucker should wait until all dcimg files for a series are
+            available before spooling (True).
+
+        Returns
+        -------
+
+        """
+        # compute the whole series name
+        mdfilename = os.path.join(self.folder, mdFile)
+        self.spooler.OnNewSeries(mdfilename)
+
+        # calculate the stub with which all files start
+        series_stub = mdfilename.strip('.json')
+
+        # calculate event and zsteps filenames from stub
+        events_filename = series_stub + '_events.json'
+        zsteps_filename = series_stub + '_zsteps.json'
+
+        # which chunks from this series have we already spooled? We do not want to spool them again
+        spooled_chunks = []
+
+        spooling_complete = False
+        spool_timeout = time.time() + self.timeout
+        while (not spooling_complete) and (time.time() < spool_timeout):
+            # check to see if we have an events file (our end signal)
+            on_disk = (os.path.exists(events_filename) or os.path.exists(zsteps_filename))
+            if on_disk:
+                # flag as complete so we exit the loop after spooling
+                spooling_complete = True
+            if only_spool_complete and (not spooling_complete):
+                # keep waiting
+                time.sleep(0.1)
+                continue
+
+            # find chunks for this metadata file (this should return the full paths)
+            chunkList = sorted(glob.glob(series_stub + '*.dcimg'))
+
+            for chunk in [c for c in chunkList if c not in spooled_chunks]:
+                try:
+                    spooled_chunks.append(chunk)
+                    self.spooler.OnDCIMGChunkDetected(chunk)
+                    # time.sleep(.5)  # FIXME: When using HUFFMANCODE more frames are lost without waiting
+                    # wait until we've sent everything
+                    # this is a bit of a hack
+                    while not self.spooler.spooler.finished():
+                        time.sleep(.1)
+                finally:
+                    if delete_after_spool:
+                        # TODO: update this to only delete files if they are sent successfully
+                        self.dumpster_raccoon.eat_trash(chunk)
+
+
+
+        # we have seen our events file, the current series is complete
+        try:
+            self.spooler.OnSeriesComplete(events_filename, zsteps_filename, pushTaskToCluster=True)
+            logger.debug('Finished spooling series %s' % series_stub)
+        finally:
+            if delete_after_spool:
+                self.dumpster_raccoon.dumpster_fire()
+                self.dumpster_raccoon.eat_trash(mdfilename)
+                self.dumpster_raccoon.eat_trash(events_filename)
+                self.dumpster_raccoon.eat_trash(zsteps_filename)
+
+    def searchAndHuck(self, only_spool_new=False, delete_after_spool=False, only_spool_complete=False):
+        """
+        Monitors a folder (self.folder) and spool series which are written there and saved as dcimg files to the cluster
+
+        Parameters
+        ----------
+        only_spool_new : bool
+            Flag to determine whether the FileChucker should ignore existing files (True) or spool them (False).
+        delete_after_spool : bool
+            Flag to determine whether files are deleted after spooling (True) or spooled without deletion (False)
+        only_spool_complete : bool
+            Flag to determine whether dcimg files should be spooled as soon as they are written (False), i.e. before an
+            events file is written, or whether the FileChucker should wait until all dcimg files for a series are
+            available before spooling (True).
+
+        Returns
+        -------
+
+        """
+        md_candidates = glob.glob(os.path.join(self.folder, '*.json'))
+        #changed it to just use a list comprehension as this will be much easier to read (and there is no performance advantage to using arrays) - DB
+        metadataFiles = [f for f in md_candidates if not (f.endswith('_events.json') or f.endswith('_zsteps.json'))]
+
+
+        if not only_spool_new:
+            for mdFile in metadataFiles:
+                mdPath = os.path.join(self.folder, mdFile)
+                try:
+                    self._spoolSeries(mdPath, delete_after_spool=delete_after_spool)
+                except Exception:
+                    logger.exception(traceback.format_exc())
+                except KeyboardInterrupt:
+                    sys.exit()
+
+        #ignore metadata files corresponding to series we have already spooled
+        ignoreList = set(metadataFiles)
+
+        while True: #NB!!!!: this will run for ever
+            # search for new files, use sets for sake of speed
+            md_candidates = set(glob.glob(self.folder + '\*.json'))
+            # keep ignoreList from growing unnecessarily; only track files that still exist
+            ignoreList &= md_candidates
+            # generate set of only the basic metadata files
+            chaff = [f for f in md_candidates if (f.endswith('_events.json') or f.endswith('_zsteps.json'))]
+            metadataFiles = md_candidates.difference(chaff).difference(ignoreList)
+            try:
+                # check if there are any series to spool
+                if len(metadataFiles) < 1:
+                    raise IndexError
+
+                # add files about to be spooled to the ignore list
+                ignoreList |= metadataFiles
+                # spool series
+                for mdFile in metadataFiles:
+                    self.huck(mdFile, delete_after_spool, only_spool_complete)
+
+            except IndexError:
+                #this happens if there are no new metadata files - wait until there are some.
+                time.sleep(.1) #wait just long enough to stop us from being in a CPU busy loop
+                pass
+            except KeyboardInterrupt:
+                sys.exit() #make sure other threads get killed if we kill with ctrl-c
+            except Exception:
+                logger.exception(traceback.format_exc())
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-n', dest='only_spool_new', action='store_true',
+                        help='Only spool new files as they are saved')
+    parser.add_argument('-d', dest='delete_after_spool', action='store_true',
+                        help='Delete files after they are spooled to the cluster')
+    parser.add_argument('-c', dest='only_spool_complete', action='store_true',
+                        help='spool series as dcimg files are written, rather than waiting for the entire series')
+    parser.add_argument('-q', dest='quantize', action='store_true',
+                        help='Quantize with sqrt(N) interval scaling')
+    parser.add_argument('testFolder', metavar='testFolder', type=str,
+                        help='Folder for fileChucker to monitor')
+    parser.add_argument('-t', '--timeout', type=float, default=12.,
+                        help='Time to wait for single series to spool')
+    args = parser.parse_args()
+
+    searcher = venerableFileChucker(args.testFolder, timeout=args.timeout, quantize=args.quantize)
+    searcher.searchAndHuck(args.only_spool_new, args.delete_after_spool, args.only_spool_complete)

PYME/experimental/dcimgSpoolShim.py

@@ -0,0 +1,174 @@
+import json
+import os
+import time
+
+from PYME.IO import HTTPSpooler_v2 as HTTPSpooler
+from PYME.Analysis import MetaData
+from PYME.IO import MetaDataHandler, clusterIO, unifiedIO
+from PYME.IO.DataSources import DcimgDataSource, MultiviewDataSource
+from PYME.IO.clusterExport import ImageFrameSource, MDSource
+from PYME.cluster import HTTPRulePusher
+
+# DT is the period of time the spooler will wait before checking if a file is free to be spooled (i.e. external
+# acquisition software has finished writing the dcimg file)
+DT = 0.1
+
+def _writing_finished(filename):
+    """Check to see whether anyone else has this file open.
+
+    the check is performed by attempting to rename the file"""
+
+    try:
+        #rename the file and then rename back. This will fail if
+        #another process has the file open
+        os.rename(filename, filename + '_')
+        os.rename(filename + '_', filename)
+        return True
+    except:
+        return False
+
+def _wait_for_file(filename, timeout=20):
+    """
+    Wait until others have finished with the file.
+
+    WARNING/FIXME: This can block for ever. Set the up limit of waiting to 20 seconds by default
+
+    Parameters
+    ----------
+    filename : str
+        The fully resolved path of the file in question
+    timeout : int or float
+        Time period to keep checking whether the file is written before giving up
+
+    Returns
+    -------
+    finished : bool
+        True if file is no longer held by external programs, False if still in use by other programs after the timeout.
+
+    """
+    stop = time.time() + timeout
+    while not _writing_finished(filename):
+        time.sleep(DT)
+        if time.time() >= stop:
+            return False
+    return True
+
+
+class DCIMGSpoolShim(object):
+    """
+    DCIMGSpoolShim provides methods to interface between DcimgDataSource and HTTPSpooler, so that one can spool
+    dcimg files (containing arbitary numbers of image frames) as they are finished writing.
+    """
+    def __init__(self, timeout):
+        self.n_spooled = 0
+        self.timeout = timeout
+    
+    def OnNewSeries(self, metadataFilename, comp_settings=None):
+        """Called when a new series is detected (ie the <seriesname>.json)
+        file is detected
+        """
+        if comp_settings is None:
+            comp_settings = {}
+        # Make sure that json file is done writing
+        success = _wait_for_file(metadataFilename, self.timeout)
+        if not success:
+            raise UserWarning('dcimg file is taking too long to finish writing')
+
+
+        #create an empty metadatahandler
+        self.mdh = MetaDataHandler.NestedClassMDHandler(MetaData.BareBones)
+
+        #load metadata from file and insert into our metadata handler
+        with open(metadataFilename, 'r') as f:
+            mdd = json.load(f)
+            self.mdh.update(mdd)
+
+        #determine a filename on the cluster from our local filename
+        #TODO - make this more complex to generate suitable directory structures
+        filename = unifiedIO.verbose_fix_name(os.path.splitext(metadataFilename)[0])
+        
+        dirname, seriesname = os.path.split(filename)
+        
+        #Strip G:\\ in filename to test if it caused connection problem to some nodes in cluster
+        dirname = dirname[dirname.find('\\') + 1:]
+        
+        cluster_filename = os.path.join(dirname, '%03d' % int(self.n_spooled/1000), seriesname)
+        
+        #create virtual frame and metadata sources
+        self.imgSource = ImageFrameSource()
+        self.metadataSource = MDSource(self.mdh)
+        MetaDataHandler.provideStartMetadata.append(self.metadataSource)
+
+        #generate the spooler
+        comp_settings.update({'quantizationOffset': self.mdh.getOrDefault('Camera.ADOffset', 0)})
+        self.spooler = HTTPSpooler.Spooler(cluster_filename, self.imgSource.onFrame, frameShape=None,
+                                           compressionSettings=comp_settings)
+
+        self.n_spooled += 1
+        #spool our data
+        self.spooler.StartSpool()
+
+    def OnDCIMGChunkDetected(self, chunkFilename):
+        """Called whenever a new chunk is detected.
+        spools that chunk to the cluster"""
+        success = _wait_for_file(chunkFilename, self.timeout)
+        if not success:
+            raise UserWarning('dcimg file is taking too long to finish writing')
+
+        chunk = DcimgDataSource.DataSource(chunkFilename)
+        croppedChunk = MultiviewDataSource.DataSource(chunk, self.mdh)
+
+        self.imgSource.spoolData(croppedChunk)
+        self.spooler.FlushBuffer()
+
+    def OnSeriesComplete(self, eventsFilename=None, zstepsFilename=None, pushTaskToCluster=False):
+        """Called when the series is finished (ie we have seen)
+        the events file"""
+
+        if (not eventsFilename is None) and (os.path.exists(eventsFilename)):
+            # Update event Log with events.json
+            with open(eventsFilename, 'r') as f:
+                 events = json.load(f)
+
+            for evt in events:
+                name, descr, timestamp = evt
+                self.spooler.evtLogger.logEvent(eventName=name, eventDescr=descr, timestamp=float(timestamp))
+
+        if (not zstepsFilename is None) and (os.path.exists(zstepsFilename)):
+            #create pseudo events based on our zstep information
+            with open(zstepsFilename, 'r') as f:
+                zsteps = json.load(f)
+
+            positions = zsteps['PIFOC_positions']
+            startFrames = zsteps['Start_Frame_eachZ']
+
+            startTime = self.mdh.getOrDefault('StartTime', 0)
+            cycleTime = self.mdh.getOrDefault('Camera.CycleTime', 0.01) #use a default frame length of 10 ms. Not super critical
+
+            for pos, fr in zip(positions, startFrames):
+                fakeTime = startTime + cycleTime*fr
+                self.spooler.evtLogger.logEvent(eventName='StartAq', eventDescr='%d' % fr, timestamp=fakeTime)
+                self.spooler.evtLogger.logEvent(eventName='ProtocolFocus', eventDescr='%d, %3.3f' % (fr, pos),
+                                                timestamp=fakeTime)
+        
+        self.spooler.StopSpool()
+        self.spooler.FlushBuffer()
+
+        if pushTaskToCluster:
+
+            self.mdh.setEntry('Analysis.BGRange', [-32, 0])
+            self.mdh.setEntry('Analysis.DebounceRadius', 4)
+            self.mdh.setEntry('Analysis.DetectionThreshold', 0.75)
+            self.mdh.setEntry('Analysis.FiducialThreshold', 1.8)
+            self.mdh.setEntry('Analysis.FitModule', 'AstigGaussGPUFitFR')
+            self.mdh.setEntry('Analysis.PCTBackground', 0.25)
+            self.mdh.setEntry('Analysis.ROISize', 7.5)
+            self.mdh.setEntry('Analysis.StartAt', 32)
+            self.mdh.setEntry('Analysis.TrackFiducials', False)
+            self.mdh.setEntry('Analysis.subtractBackground', True)
+            self.mdh.setEntry('Analysis.GPUPCTBackground', True)
+            cluster_filename = 'pyme-cluster://%s/%s' % (clusterIO.local_serverfilter, self.spooler.seriesName)
+            HTTPRulePusher.launch_localize(analysisMDH=self.mdh, seriesName=cluster_filename)
+
+        #remove the metadata generator
+        MetaDataHandler.provideStartMetadata.remove(self.metadataSource)

PYME/experimental/dcimg_chuck_test.py

@@ -0,0 +1,78 @@
+##########################################################
+
+"""
+Script to test the cluster side of dcimg spooling independantly of labview.
+Copies a single series repeatedly to the spool directory whilst incrementally increasing the filename.
+
+Arguments:
+      template_loc : a directory containing template files
+      output_loc : the directory to spool to
+
+The template directory should contain 3 files:
+
+XXX.json
+XXX_Chan001.dcimg
+XXX_zsteps.json
+
+"""
+
+import os
+import sys
+import shutil
+import time
+
+N_SERIES = 10000 # number of series to spool
+SERIES_DELAY_S = 8 #delay between series
+
+
+def main():
+    template_loc = sys.argv[1]
+    output_loc = sys.argv[2]
+
+    start_i = 0
+    if len(sys.argv) > 3:
+        start_i = int(sys.argv[3])
+    
+    template_json = os.path.join(template_loc, 'XXX.json')
+    template_dcimg = os.path.join(template_loc, 'XXX.dcimg')
+    template_zsteps = os.path.join(template_loc, 'XXX_zsteps.json')
+
+    #print template_loc, output_loc
+
+    with open(template_json, 'rb') as f:
+        json_data = f.read()
+
+    with open(template_dcimg, 'rb') as f:
+        dcimg_data = f.read()
+
+    with open(template_zsteps, 'rb') as f:
+        zsteps_data = f.read()
+    
+    t_last = time.time()
+    time.sleep(0.1)
+    for i in range(start_i, N_SERIES):
+        t_now = time.time()
+        delay = SERIES_DELAY_S - max(t_now - t_last, 0)
+        print('delay: %f' % delay)
+        if delay > 0:
+            time.sleep(delay)
+        print('spooling series %d' % i)
+        #shutil.copy(template_json, os.path.join(output_loc, 'Series%04d.json' % i))
+        with open(os.path.join(output_loc, 'Series%04d.json' % i), 'wb') as f:
+            f.write(json_data)
+
+        #note that this is slowish (probably due to having to allocate lots of ram). Might be quicker to read in once and write repeatedly.
+        #shutil.copy(template_dcimg, os.path.join(output_loc, 'Series%04d_ch001.dcimg' % i))
+        with open(os.path.join(output_loc, 'Series%04d_ch001.dcimg' % i), 'wb') as f:
+            f.write(dcimg_data)
+
+        #shutil.copy(template_zsteps, os.path.join(output_loc, 'Series%04d_zsteps.json' % i))
+        with open(os.path.join(output_loc, 'Series%04d_zsteps.json' % i), 'wb') as f:
+            f.write(zsteps_data)
+
+        print('Sppoled series %d' % i)
+        t_last = t_now
+
+
+if __name__ == '__main__':
+    main()