PyOPIA 2.3.0__tar.gz → 2.4.1__tar.gz

{pyopia-2.3.0 → pyopia-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyOPIA
-Version: 2.3.0
+Version: 2.4.1
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Home-page: https://github.com/sintef/pyopia
 Keywords: Ocean,Particles,Imaging,Measurement,Size distribution

pyopia-2.4.1/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.4.1'

{pyopia-2.3.0 → pyopia-2.4.1}/pyopia/background.py

@@ -8,13 +8,19 @@ def ini_background(bgfiles, load_function):
     '''
     Create and initial background stack and average image
 
-    Args:
-        bgfiles (list)                   : list of strings of filenames to be used in background creation
-        load_function (function object)  : this function should take a filename and return an image,
-                                           for example: :func:`pyopia.instrument.silcam.load_image`
-    Returns:
-        bgstack (list)              : list of all images in the background stack
-        imbg (array)                : background image
+    Parameters:
+    -----------
+    bgfiles : list
+        List of strings of filenames to be used in background creation
+    load_function : object
+        This function should take a filename and return an image, for example: :func:`pyopia.instrument.silcam.load_image`
+
+    Returns
+    -------
+    bgstack : list
+        list of all images in the background stack
+    imbg : array
+        background image
     '''
     bgstack = []
     for f in bgfiles:
@@ -33,14 +39,21 @@ def shift_bgstack_accurate(bgstack, imbg, imnew):
     The new background is calculated slowly by computing the mean of all images
     in the background stack.
 
-    Args:
-        bgstack (list)      : list of all images in the background stack
-        imbg (uint8)        : background image
-        imnew (unit8)       : new image to be added to stack
-
-    Returns:
-        bgstack (updated list of all background images)
-        imbg (updated actual background image)
+    Parameters:
+    -----------
+    bgstack : list
+        list of all images in the background stack
+    imbg : array
+        background image
+    imnew : array
+        new image to be added to stack
+
+    Returns
+    -------
+    bgstack : list
+        updated list of all background images
+    imbg : array
+        updated actual background image
     '''
     bgstack.pop(0)  # pop the oldest image from the stack,
     bgstack.append(imnew)  # append the new image to the stack
@@ -56,14 +69,21 @@ def shift_bgstack_fast(bgstack, imbg, imnew):
     adding the new image (both scaled by the stacklength).
     This is close to a running mean, but not quite.
 
-    Args:
-        bgstack (list)      : list of all images in the background stack
-        imbg (uint8)        : background image
-        imnew (unit8)       : new image to be added to stack
-
-    Returns:
-        bgstack (updated list of all background images)
-        imbg (updated actual background image)
+    Parameters:
+    -----------
+    bgstac : list
+        list of all images in the background stack
+    imbg : uint8
+        background image
+    imnew : unit8
+        new image to be added to stack
+
+    Returns
+    -------
+    bgstack : list
+        updated list of all background images
+    imbg : array
+        updated actual background image
     '''
     stacklength = len(bgstack)
     imold = bgstack.pop(0)  # pop the oldest image from the stack,
@@ -82,14 +102,17 @@ def correct_im_accurate(imbg, imraw):
     There is a small chance of clipping of imc in both crushed blacks and blown
     highlights if the background or raw images are very poorly obtained
 
-    Args:
-      imbg (float64)  : background averaged image
-      imraw (float64) : raw image
-      imbg (float64)  : background averaged image
-      imraw (float64) : raw image
-
-    Returns:
-      im_corrected (float64)   : corrected image, same type as input
+    Parameters:
+    -----------
+    imbg : float64
+        background averaged image
+    imraw : float64
+        raw image
+
+    Returns
+    -------
+    im_corrected : float64
+        corrected image, same type as input
     '''
 
     im_corrected = imraw - imbg
@@ -108,12 +131,17 @@ def correct_im_fast(imbg, imraw):
     There is high potential for clipping of imc in both crushed blacks an blown
     highlights, especially if the background or raw images are not properly obtained
 
-    Args:
-      imraw (float64) : raw image
-      imbg (float64)  : background averaged image
-
-    Returns:
-      im_corrected (float64)   : corrected image
+    Parameters:
+    -----------
+    imraw : array
+        raw image
+    imbg : array
+        background averaged image
+
+    Returns
+    -------
+    im_corrected : array
+        corrected image
     '''
     im_corrected = imraw - imbg
 
@@ -130,18 +158,27 @@ def shift_and_correct(bgstack, imbg, imraw, stacklength, real_time_stats=False):
 
     This is a wrapper for shift_bgstack and correct_im
 
-    Args:
-        bgstack (list)                  : list of all images in the background stack
-        imbg (float64)                  : background image
-        imraw (float64)                 : raw image
-        stacklength (int)               : unsed int here - just there to maintain the same behaviour as
-                                          shift_bgstack_fast()
-        real_time_stats=False (Bool)    : if True use fast functions, if False use accurate functions
-
-    Returns:
-        bgstack (list)                  : list of all images in the background stack
-        imbg (float64)                  : background averaged image
-        im_corrected (float64)          : corrected image
+    Parameters:
+    -----------
+    bgstack : list
+        list of all images in the background stack
+    imbg : float64
+        background image
+    imraw : float64
+        raw image
+    stacklength : int
+        unused int here - just there to maintain the same behaviour as shift_bgstack_fast()
+    real_time_stats : Bool, optional
+        True use fast functions, if False use accurate functions., by default False
+
+    Returns
+    -------
+    bgstack : list
+        list of all images in the background stack
+    imbg : float64
+        background averaged image
+    im_corrected : float64
+        corrected image
     '''
 
     if real_time_stats:

{pyopia-2.3.0 → pyopia-2.4.1}/pyopia/cli.py

@@ -4,13 +4,14 @@ PyOPIA top-level code primarily for managing cmd line entry points
 
 import typer
 import toml
-from glob import glob
 import os
 import datetime
 import traceback
 import logging
 from rich.progress import track, Progress
+from rich.logging import RichHandler
 import pandas as pd
+import threading
 
 import pyopia.background
 import pyopia.instrument.silcam
@@ -105,23 +106,35 @@ def generate_config(instrument: str, raw_files: str, model_path: str, outfolder:
 
 
 @app.command()
-def process(config_filename: str):
+def process(config_filename: str, num_chunks: int = 1):
     '''Run a PyOPIA processing pipeline based on given a config.toml
+
+    Parameters
+    ----------
+    config_filename : str
+        config filename
+    numchunks : int, optional
+        split the dataset into chucks, and process in parallell, by default 1
+
     '''
     from pyopia.io import load_toml
     from pyopia.pipeline import Pipeline
 
-    logger = logging.getLogger()
-
     with Progress(transient=True) as progress:
         progress.console.print("[blue]LOAD CONFIG")
         pipeline_config = load_toml(config_filename)
 
         setup_logging(pipeline_config)
+        logger = logging.getLogger('rich')
+        logger.info(f'PyOPIA process started {pd.Timestamp.now()}')
+
+        check_chunks(num_chunks, pipeline_config)
 
         progress.console.print("[blue]OBTAIN FILE LIST")
-        files = sorted(glob(pipeline_config['general']['raw_files']))
-        nfiles = len(files)
+        raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
+        average_window = pipeline_config['steps']['correctbackground'].get('average_window', 0)
+        bgshift_function = pipeline_config['steps']['correctbackground'].get('bgshift_function', 'pass')
+        raw_files.prepare_chunking(num_chunks, average_window, bgshift_function)
 
         progress.console.print('[blue]PREPARE FOLDERS')
         if 'output' not in pipeline_config['steps']:
@@ -134,20 +147,32 @@ def process(config_filename: str):
         if os.path.isfile(output_datafile + '-STATS.nc'):
             dt_now = datetime.datetime.now().strftime('D%Y%m%dT%H%M%S')
             newname = output_datafile + '-conflict-' + str(dt_now) + '-STATS.nc'
-            progress.console.print('[red]Renaming conflicting file to: ' +
-                                   newname)
+            logger.warning(f'Renaming conflicting file to: {newname}')
             os.rename(output_datafile + '-STATS.nc', newname)
 
         progress.console.print("[blue]INITIALISE PIPELINE")
-        processing_pipeline = Pipeline(pipeline_config)
 
-    for filename in track(files, description=f'[blue]Processing progress through {nfiles} files:'):
-        try:
-            processing_pipeline.run(filename)
-        except Exception as e:
-            progress.console.print("[red]An error occured in processing, skipping rest of pipeline and moving to next image.")
-            logger.error(e)
-            logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+    def process_file_list(file_list, c):
+        processing_pipeline = Pipeline(pipeline_config)
+        for filename in track(file_list, description=f'[blue]Processing progress (chunk {c})',
+                              disable=c != 0):
+            try:
+                logger.debug(f'Chunk {c} starting to process {filename}')
+                processing_pipeline.run(filename)
+            except Exception as e:
+                logger.warning('[red]An error occured in processing, ' +
+                               'skipping rest of pipeline and moving to next image.' +
+                               f'(chunk {c})')
+                logger.error(e)
+                logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+
+    # With one chunk we keep the non-threaded functionality to ensure backwards compatibility
+    if num_chunks == 1:
+        process_file_list(raw_files, 0)
+    else:
+        for c, chunk in enumerate(raw_files.chunked_files):
+            job = threading.Thread(target=process_file_list, args=(chunk, c, ))
+            job.start()
 
 
 @app.command()
@@ -179,15 +204,25 @@ def setup_logging(pipeline_config):
     log_file = pipeline_config['general'].get('log_file', None)
     log_level_name = pipeline_config['general'].get('log_level', 'INFO')
     log_level = getattr(logging, log_level_name)
-    print(log_level_name, log_level, log_file)
+
+    # Either log to file (silent console) or to console with Rich
+    if log_file is None:
+        handlers = [RichHandler(show_time=True, show_level=False)]
+    else:
+        handlers = [logging.FileHandler(log_file, mode='a')]
 
     # Configure logger
-    log_format = '%(asctime)s %(levelname)s [%(module)s.%(funcName)s] %(message)s'
-    logging.basicConfig(level=log_level, format=log_format, filename=log_file,
-                        datefmt='%Y-%m-%d %H:%M:%S')
+    log_format = '%(asctime)s %(levelname)s %(threadName)s [%(module)s.%(funcName)s] %(message)s'
+    logging.basicConfig(level=log_level, datefmt='%Y-%m-%d %H:%M:%S', format=log_format, handlers=handlers)
+
+
+def check_chunks(chunks, pipeline_config):
+    if chunks < 1:
+        raise RuntimeError('You must have at least 1 chunk')
 
-    logger = logging.getLogger()
-    logger.info(f'PyOPIA process started {pd.Timestamp.now()}')
+    append_enabled = pipeline_config['steps']['output'].get('append', True)
+    if chunks > 1 and append_enabled:
+        raise RuntimeError('Output mode must be set to "append = false" in "output" step when using more than one chunk')
 
 
 if __name__ == "__main__":

{pyopia-2.3.0 → pyopia-2.4.1}/pyopia/io.py

@@ -32,18 +32,24 @@ def write_stats(stats,
     Writes particle stats into the ouput file.
     Appends if file already exists.
 
-    Args:
-        datafilename (str)                      :   Filame prefix for -STATS.h5 file that may or may not include a path
-        stats (DataFrame or xr.Dataset)         :   STATS dataframe
-        export_name_len (int)                   :   Max number of chars allowed for col 'export name'
-        append (bool)                           :   Append all processed data into one nc file.
-                                                    Defaults to True.
-                                                    If False, then one nc file will be generated per raw image,
-                                                    which can be loaded using :func:`pyopia.io.combine_stats_netcdf_files`
-                                                    This is useful for larger datasets,
-                                                    where appending causes substantial slowdown
-                                                    as the dataset gets larger.
-        image_stats (xr.Dataset)                :   image_stats data
+    Parameters
+    ----------
+    datafilename : str
+        Filame prefix for -STATS.h5 file that may or may not include a path
+    stats : DataFrame or xr.Dataset
+        particle statistics
+    export_name_len : int
+        Max number of chars allowed for col 'export name'
+    append : bool
+        Append all processed data into one nc file.
+        Defaults to True.
+        If False, then one nc file will be generated per raw image,
+        which can be loaded using :func:`pyopia.io.combine_stats_netcdf_files`
+        This is useful for larger datasets,
+        where appending causes substantial slowdown
+        as the dataset gets larger.
+    image_stats : xr.Dataset
+        summary statistics of each raw image (including those with no particles)
     '''
 
     if len(stats) == 0:  # to avoid issue with wrong time datatypes in xarray
@@ -99,7 +105,7 @@ def make_xstats(stats, toml_steps):
 
     Returns
     -------
-    xarray.DataSet
+    xstats : xarray.Dataset
         Xarray version of stats dataframe, including metadata
     '''
     xstats = stats.to_xarray()
@@ -120,8 +126,8 @@ def load_image_stats(datafilename):
 
     Returns
     -------
-    xarray.DataArray
-        image_stats
+    image_stats : xarray.Dataset
+        summary statistics of each raw image (including those with no particles)
     '''
     with xarray.open_dataset(datafilename, engine=NETCDF_ENGINE, group='image_stats') as image_stats:
         image_stats.load()
@@ -138,7 +144,7 @@ def load_stats(datafilename):
 
     Returns
     -------
-    DataFrame
+    stats : DataFrame
         STATS DataFrame  / xarray dataset
     '''
 
@@ -170,8 +176,10 @@ def combine_stats_netcdf_files(path_to_data, prefix='*'):
 
     Returns
     -------
-    tuple
-        xstats STATS xarray dataset, image_stats dataset
+    xstats : xarray.Dataset
+        Particle statistics and metatdata from processing steps
+    image_stats : xarray.Dataset
+        summary statistics of each raw image (including those with no particles)
     '''
 
     sorted_filelist = sorted(glob(os.path.join(path_to_data, prefix + 'Image-D*-STATS.nc')))
@@ -234,7 +242,7 @@ def steps_from_xstats(xstats):
 
     Returns
     -------
-    dict
+    steps : dict
         TOML-formatted dictionary of pipeline steps
     '''
     steps = toml.loads(xstats.__getattr__('steps'))
@@ -251,7 +259,7 @@ def load_stats_as_dataframe(stats_file):
 
     Returns
     -------
-    DataFrame
+    stats : DataFrame
         stats pandas dataframe
     '''
     # obtain particle statistics from the stats file
@@ -269,8 +277,10 @@ def show_h5_meta(h5file):
     '''
     prints metadata from an exported hdf5 file created from pyopia.process
 
-    Args:
-        h5file              : h5 filename from exported data from pyopia.process
+    Parameters
+    ----------
+    h5file : str
+        h5 filename from exported data from pyopia.process
     '''
 
     with h5py.File(h5file, 'r') as f:
@@ -286,20 +296,26 @@ class StatsToDisc():
 
     Replaces the old StatsH5 class
 
-    Args:
-        output_datafile (str): prefix path for output nc file
-        dataformat (str): either 'nc' or 'h5
-        append (bool): if to allow append to an existing STATS file. Defaults to True
-        export_name_len (int): max number of chars allowed for col 'export name'. Defaults to 40
-        append (bool):          Append all processed data into one nc file.
-                                Defaults to True.
-                                If False, then one nc file will be generated per raw image,
-                                which can be loaded using :func:`pyopia.io.combine_stats_netcdf_files`
-                                This is useful for larger datasets, where appending causes substantial slowdown
-                                as the dataset gets larger.
-
-    Returns:
-        data (dict): data from pipeline
+    Parameters
+    ----------
+    output_datafile : str
+        prefix path for output nc file
+    dataformat : str
+        either 'nc' or 'h5
+    export_name_len : int
+        max number of chars allowed for col 'export name'. Defaults to 40
+    append : bool
+        Append all processed data into one nc file.
+        Defaults to True.
+        If False, then one nc file will be generated per raw image,
+        which can be loaded using :func:`pyopia.io.combine_stats_netcdf_files`
+        This is useful for larger datasets, where appending causes substantial slowdown
+        as the dataset gets larger.
+
+    Returns
+    -------
+    data : dict
+        data from pipeline
 
     Example config for pipeline useage:
 

{pyopia-2.3.0 → pyopia-2.4.1}/pyopia/pipeline.py

@@ -10,6 +10,8 @@ from operator import methodcaller
 import sys
 from pyopia.io import steps_from_xstats as steps_from_xstats # noqa: E(F401)
 import logging
+from glob import glob
+import numpy as np
 
 logger = logging.getLogger()
 
@@ -82,11 +84,15 @@ class Pipeline():
     def run(self, filename):
         '''Method for executing the processing pipeline.
 
-        Args:
-            filename (str): file to be processed
+        Parameters
+        ----------
+        filename : str
+            file to be processed
 
-        Returns:
-            stats (DataFrame): stats DataFrame of particle statistics associated with 'filename'
+        Returns
+        -------
+        stats : DataFrame
+            particle statistics associated with 'filename'
 
         Note: the returned stats from this function are single-image only and not appended
         if you loop through several filenames! It is recommended to use this step in the pipeline
@@ -257,11 +263,15 @@ class Data(TypedDict):
 def steps_to_string(steps):
     '''Deprecated. Convert pipeline steps dictionary to a human-readable string
 
-    Args:
-        steps (dict): pipeline steps dictionary
+    Parameters
+    ----------
+    steps : dict
+        pipeline steps dictionary
 
-    Returns:
-        str: human-readable string of the types and variables
+    Returns
+    -------
+    steps_str : str
+        human-readable string of the types and variables
     '''
 
     steps_str = '\n'
@@ -322,3 +332,88 @@ def build_steps(toml_steps):
         steps[step_name] = build_repr(toml_steps, step_name)
 
     return steps
+
+
+class FilesToProcess:
+    def __init__(self, glob_pattern=None):
+        '''Build file list from glob pattern if specified.
+           Create FilesToProcess.chunked_files is chunks specified
+           File list from glob will be sorted.
+
+        Parameters
+        ----------
+        glob_pattern : str, optional
+            Glob pattern, by default None
+        '''
+        self.files = None
+        self.background_files = []
+        self.chunked_files = []
+        if glob_pattern is not None:
+            self.files = sorted(glob(glob_pattern))
+
+    def from_filelist_file(self, path_to_filelist):
+        '''
+        Initialize explicit list of files to process from a text file.
+        The text file should contain one path to an image per line, which should be processed in order.
+        '''
+        with open(path_to_filelist, 'r') as fh:
+            self.files = list(fh.readlines())
+
+    def to_filelist_file(self, path_to_filelist):
+        '''Write file list to a txt file
+
+        Parameters
+        ----------
+        path_to_filelist : str
+            Path to txt file to write
+        '''
+        with open(path_to_filelist, 'w') as fh:
+            [fh.writelines(L + '\n') for L in self.files]
+
+    def prepare_chunking(self, num_chunks, average_window, bgshift_function):
+        if num_chunks > len(self.files) // 2:
+            raise RuntimeError('Number of chunks exceeds more than half the number of files to process. Use less chunks.')
+        self.chunk_files(num_chunks)
+        self.build_initial_background_files(average_window=average_window)
+        self.insert_bg_files_into_chunks(bgshift_function=bgshift_function)
+
+    def chunk_files(self, num_chunks: int):
+        '''Chunk the file list and create FilesToProcess.chunked_files
+
+        Parameters
+        ----------
+        chunks : int
+            number of chunks to produce (must be at least 1)
+        '''
+        if num_chunks < 1:
+            raise RuntimeError('You must have at least one chunk')
+        chunk_length = int(np.ceil(len(self.files) / num_chunks))
+        self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)]
+
+    def insert_bg_files_into_chunks(self, bgshift_function='pass'):
+        average_window = len(self.background_files)
+        for i, chunk in enumerate(self.chunked_files):
+            if i > 0 and bgshift_function != 'pass':
+                # If the bgshift_function is not pass then we need to find a new set of
+                # background images for the start of next chunk. These will be the last
+                # average_window number of files from the previous chunk.
+                # If bgshift_function is 'pass', then we should use the same background files for all chunks
+                # so there is no need to extend the list of background files here
+                self.background_files.extend(self.chunked_files[i-1][-average_window:])
+            # we have to loop backwards over bg_files because we are inserting into the top of the chunk
+            chunk = [chunk.insert(0, bg_file) for bg_file in reversed(self.background_files[-average_window:])]
+
+    def build_initial_background_files(self, average_window=0):
+        '''
+        Create a list of files to use for initializing the background in the first chunk
+        '''
+        self.background_files = []
+        for f in self.files[0:average_window]:
+            self.background_files.append(f)
+
+    def __len__(self):
+        return len(self.files)
+
+    def __iter__(self):
+        for filename in self.files:
+            yield filename

{pyopia-2.3.0 → pyopia-2.4.1}/pyopia/plotting.py

@@ -9,12 +9,14 @@ import numpy as np
 
 
 def show_image(image, pixel_size):
-    '''
-    Plots a scaled figure (in mm) of an image
-
-    Args:
-        image (float) : Image (usually a corrected image, such as im_corrected)
-        pixel_size (float) : the pixel size (um) of the imaging system used
+    '''Plots a scaled figure (in mm) of an image
+
+    Parameters
+    ----------
+    image : float
+        Image (usually a corrected image, such as im_corrected)
+    pixel_size : float
+        the pixel size (um) of the imaging system used
     '''
     r, c = np.shape(image[:, :, 0])
 
@@ -31,10 +33,12 @@ def montage_plot(montage, pixel_size):
     '''
     Plots a SilCam particle montage with a 1mm scale reference
 
-    Args:
-        montage (uint8)    : a SilCam montage created with scpp.make_montage
-        pixel_size (float) : the pixel size of the SilCam used, obtained from settings.PostProcess.pix_size in the
-                             config ini file
+    Parameters
+    ----------
+    montage : uint8
+        a montage created with scpp.make_montage
+    pixel_size : float
+        the pixel size (um) of the imaging system used
     '''
     msize = np.shape(montage)[0]
     ex = pixel_size * np.float64(msize) / 1000.