PyOPIA 2.2.0__tar.gz → 2.4.0__tar.gz

{pyopia-2.2.0 → pyopia-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyOPIA
-Version: 2.2.0
+Version: 2.4.0
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Home-page: https://github.com/sintef/pyopia
 Keywords: Ocean,Particles,Imaging,Measurement,Size distribution

pyopia-2.4.0/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.4.0'

{pyopia-2.2.0 → pyopia-2.4.0}/pyopia/cli.py

@@ -4,17 +4,19 @@ PyOPIA top-level code primarily for managing cmd line entry points
 
 import typer
 import toml
-from glob import glob
 import os
 import datetime
 import traceback
 import logging
 from rich.progress import track, Progress
+from rich.logging import RichHandler
 import pandas as pd
+import threading
 
 import pyopia.background
 import pyopia.instrument.silcam
 import pyopia.instrument.holo
+import pyopia.instrument.uvp
 import pyopia.instrument.common
 import pyopia.io
 import pyopia.pipeline
@@ -80,7 +82,7 @@ def generate_config(instrument: str, raw_files: str, model_path: str, outfolder:
     Parameters
     ----------
     instrument : str
-        either `silcam` or `holo`
+        either `silcam`, `holo` or `uvp`
     raw_files : str
         raw_files
     model_path : str
@@ -95,6 +97,8 @@ def generate_config(instrument: str, raw_files: str, model_path: str, outfolder:
             pipeline_config = pyopia.instrument.silcam.generate_config(raw_files, model_path, outfolder, output_prefix)
         case 'holo':
             pipeline_config = pyopia.instrument.holo.generate_config(raw_files, model_path, outfolder, output_prefix)
+        case 'uvp':
+            pipeline_config = pyopia.instrument.uvp.generate_config(raw_files, model_path, outfolder, output_prefix)
 
     config_filename = instrument + "-config.toml"
     with open(config_filename, "w") as toml_file:
@@ -102,23 +106,35 @@ def generate_config(instrument: str, raw_files: str, model_path: str, outfolder:
 
 
 @app.command()
-def process(config_filename: str):
+def process(config_filename: str, num_chunks: int = 1):
     '''Run a PyOPIA processing pipeline based on given a config.toml
+
+    Parameters
+    ----------
+    config_filename : str
+        config filename
+    numchunks : int, optional
+        split the dataset into chucks, and process in parallell, by default 1
+
     '''
     from pyopia.io import load_toml
     from pyopia.pipeline import Pipeline
 
-    logger = logging.getLogger()
-
     with Progress(transient=True) as progress:
         progress.console.print("[blue]LOAD CONFIG")
         pipeline_config = load_toml(config_filename)
 
         setup_logging(pipeline_config)
+        logger = logging.getLogger('rich')
+        logger.info(f'PyOPIA process started {pd.Timestamp.now()}')
+
+        check_chunks(num_chunks, pipeline_config)
 
         progress.console.print("[blue]OBTAIN FILE LIST")
-        files = sorted(glob(pipeline_config['general']['raw_files']))
-        nfiles = len(files)
+        raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
+        average_window = pipeline_config['steps']['correctbackground'].get('average_window', 0)
+        bgshift_function = pipeline_config['steps']['correctbackground'].get('bgshift_function', 'pass')
+        raw_files.prepare_chunking(num_chunks, average_window, bgshift_function)
 
         progress.console.print('[blue]PREPARE FOLDERS')
         if 'output' not in pipeline_config['steps']:
@@ -131,20 +147,32 @@ def process(config_filename: str):
         if os.path.isfile(output_datafile + '-STATS.nc'):
             dt_now = datetime.datetime.now().strftime('D%Y%m%dT%H%M%S')
             newname = output_datafile + '-conflict-' + str(dt_now) + '-STATS.nc'
-            progress.console.print('[red]Renaming conflicting file to: ' +
-                                   newname)
+            logger.warning(f'Renaming conflicting file to: {newname}')
             os.rename(output_datafile + '-STATS.nc', newname)
 
         progress.console.print("[blue]INITIALISE PIPELINE")
-        processing_pipeline = Pipeline(pipeline_config)
 
-    for filename in track(files, description=f'[blue]Processing progress through {nfiles} files:'):
-        try:
-            processing_pipeline.run(filename)
-        except Exception as e:
-            progress.console.print("[red]An error occured in processing, skipping rest of pipeline and moving to next image.")
-            logger.error(e)
-            logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+    def process_file_list(file_list, c):
+        processing_pipeline = Pipeline(pipeline_config)
+        for filename in track(file_list, description=f'[blue]Processing progress (chunk {c})',
+                              disable=c != 0):
+            try:
+                logger.debug(f'Chunk {c} starting to process {filename}')
+                processing_pipeline.run(filename)
+            except Exception as e:
+                logger.warning('[red]An error occured in processing, ' +
+                               'skipping rest of pipeline and moving to next image.' +
+                               f'(chunk {c})')
+                logger.error(e)
+                logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+
+    # With one chunk we keep the non-threaded functionality to ensure backwards compatibility
+    if num_chunks == 1:
+        process_file_list(raw_files, 0)
+    else:
+        for c, chunk in enumerate(raw_files.chunked_files):
+            job = threading.Thread(target=process_file_list, args=(chunk, c, ))
+            job.start()
 
 
 @app.command()
@@ -176,15 +204,25 @@ def setup_logging(pipeline_config):
     log_file = pipeline_config['general'].get('log_file', None)
     log_level_name = pipeline_config['general'].get('log_level', 'INFO')
     log_level = getattr(logging, log_level_name)
-    print(log_level_name, log_level, log_file)
+
+    # Either log to file (silent console) or to console with Rich
+    if log_file is None:
+        handlers = [RichHandler(show_time=True, show_level=False)]
+    else:
+        handlers = [logging.FileHandler(log_file, mode='a')]
 
     # Configure logger
-    log_format = '%(asctime)s %(levelname)s [%(module)s.%(funcName)s] %(message)s'
-    logging.basicConfig(level=log_level, format=log_format, filename=log_file,
-                        datefmt='%Y-%m-%d %H:%M:%S')
+    log_format = '%(asctime)s %(levelname)s %(threadName)s [%(module)s.%(funcName)s] %(message)s'
+    logging.basicConfig(level=log_level, datefmt='%Y-%m-%d %H:%M:%S', format=log_format, handlers=handlers)
+
+
+def check_chunks(chunks, pipeline_config):
+    if chunks < 1:
+        raise RuntimeError('You must have at least 1 chunk')
 
-    logger = logging.getLogger()
-    logger.info(f'PyOPIA process started {pd.Timestamp.now()}')
+    append_enabled = pipeline_config['steps']['output'].get('append', True)
+    if chunks > 1 and append_enabled:
+        raise RuntimeError('Output mode must be set to "append = false" in "output" step when using more than one chunk')
 
 
 if __name__ == "__main__":

{pyopia-2.2.0 → pyopia-2.4.0}/pyopia/instrument/silcam.py

@@ -34,9 +34,9 @@ def load_image(filename):
     Returns
     -------
     array
-        raw image
+        raw image float between 0-1
     '''
-    img = np.load(filename, allow_pickle=False).astype(np.float64)
+    img = np.load(filename, allow_pickle=False).astype(np.float64) / 255
     return img
 
 
@@ -68,7 +68,7 @@ class SilCamLoad():
 
     def __call__(self, data):
         timestamp = timestamp_from_filename(data['filename'])
-        img = load_image(data['filename']).astype(np.float64)/255
+        img = load_image(data['filename'])
         data['timestamp'] = timestamp
         data['imraw'] = img
         return data

pyopia-2.4.0/pyopia/instrument/uvp.py

@@ -0,0 +1,127 @@
+'''
+Module containing UVP specific tools to enable compatability with the :mod:`pyopia.pipeline`
+'''
+
+import os
+import numpy as np
+import pandas as pd
+import skimage.io
+
+
+def timestamp_from_filename(filename):
+    '''get a pandas timestamp from a UVP vignette image filename
+
+    Args:
+        filename (string): UVP filename (.png)
+
+    Returns:
+        timestamp: timestamp from pandas.to_datetime()
+    '''
+
+    # get the timestamp of the image (in this case from the filename)
+    timestr = os.path.split(filename)[-1].strip('.png')
+    timestamp = pd.to_datetime(timestr)
+    return timestamp
+
+
+def load_image(filename):
+    '''load a UVP .png file from disc
+
+    Parameters
+    ----------
+    filename : string
+        filename to load
+
+    Returns
+    -------
+    array
+        raw image float between 0-1, inverted so that particles are dark on a light background
+    '''
+    img_darkfield = skimage.io.imread(filename).astype(np.float64)
+    img_inverted = (255 - img_darkfield) / 255
+    return img_inverted
+
+
+class UVPLoad():
+    '''PyOpia pipline-compatible class for loading a single UVP image
+    using :func:`pyopia.instrument.uvp.load_image`
+    and extracting the timestamp using
+    :func:`pyopia.instrument.uvp.timestamp_from_filename`
+
+    Pipeline input data:
+    ---------
+    :class:`pyopia.pipeline.Data`
+        containing the following keys:
+
+        :attr:`pyopia.pipeline.Data.filename`
+
+    Returns:
+    --------
+    :class:`pyopia.pipeline.Data`
+        containing the following new keys:
+
+        :attr:`pyopia.pipeline.Data.timestamp`
+
+        :attr:`pyopia.pipeline.Data.img`
+    '''
+
+    def __init__(self):
+        pass
+
+    def __call__(self, data):
+        timestamp = timestamp_from_filename(data['filename'])
+        img = load_image(data['filename'])
+        data['timestamp'] = timestamp
+        data['imraw'] = img
+        return data
+
+
+def generate_config(raw_files: str, model_path: str, outfolder: str, output_prefix: str):
+    '''Generate example uvp config.toml as a dict
+
+    Parameters
+    ----------
+    raw_files : str
+        raw_files
+    model_path : str
+        model_path
+    outfolder : str
+        outfolder
+    output_prefix : str
+        output_prefix
+
+    Returns:
+    --------
+    dict
+        pipeline_config toml dict
+    '''
+    # define the configuration to use in the processing pipeline - given as a dictionary - with some values defined above
+    pipeline_config = {
+        'general': {
+            'raw_files': raw_files,
+            'pixel_size': 80  # pixel size in um
+        },
+        'steps': {
+            'classifier': {
+                'pipeline_class': 'pyopia.classify.Classify',
+                'model_path': model_path
+            },
+            'load': {
+                'pipeline_class': 'pyopia.instrument.uvp.UVPLoad'
+            },
+            'segmentation': {
+                'pipeline_class': 'pyopia.process.Segment',
+                'threshold': 0.95,
+                'segment_source': 'imraw'
+            },
+            'statextract': {
+                'pipeline_class': 'pyopia.process.CalculateStats',
+                'roi_source': 'imraw'
+            },
+            'output': {
+                'pipeline_class': 'pyopia.io.StatsH5',
+                'output_datafile': os.path.join(outfolder, output_prefix)
+            }
+        }
+    }
+    return pipeline_config

{pyopia-2.2.0 → pyopia-2.4.0}/pyopia/pipeline.py

@@ -10,6 +10,8 @@ from operator import methodcaller
 import sys
 from pyopia.io import steps_from_xstats as steps_from_xstats # noqa: E(F401)
 import logging
+from glob import glob
+import numpy as np
 
 logger = logging.getLogger()
 
@@ -322,3 +324,88 @@ def build_steps(toml_steps):
         steps[step_name] = build_repr(toml_steps, step_name)
 
     return steps
+
+
+class FilesToProcess:
+    def __init__(self, glob_pattern=None):
+        '''Build file list from glob pattern if specified.
+           Create FilesToProcess.chunked_files is chunks specified
+           File list from glob will be sorted.
+
+        Parameters
+        ----------
+        glob_pattern : str, optional
+            Glob pattern, by default None
+        '''
+        self.files = None
+        self.background_files = []
+        self.chunked_files = []
+        if glob_pattern is not None:
+            self.files = sorted(glob(glob_pattern))
+
+    def from_filelist_file(self, path_to_filelist):
+        '''
+        Initialize explicit list of files to process from a text file.
+        The text file should contain one path to an image per line, which should be processed in order.
+        '''
+        with open(path_to_filelist, 'r') as fh:
+            self.files = list(fh.readlines())
+
+    def to_filelist_file(self, path_to_filelist):
+        '''Write file list to a txt file
+
+        Parameters
+        ----------
+        path_to_filelist : str
+            Path to txt file to write
+        '''
+        with open(path_to_filelist, 'w') as fh:
+            [fh.writelines(L + '\n') for L in self.files]
+
+    def prepare_chunking(self, num_chunks, average_window, bgshift_function):
+        if num_chunks > len(self.files) // 2:
+            raise RuntimeError('Number of chunks exceeds more than half the number of files to process. Use less chunks.')
+        self.chunk_files(num_chunks)
+        self.build_initial_background_files(average_window=average_window)
+        self.insert_bg_files_into_chunks(bgshift_function=bgshift_function)
+
+    def chunk_files(self, num_chunks: int):
+        '''Chunk the file list and create FilesToProcess.chunked_files
+
+        Parameters
+        ----------
+        chunks : int
+            number of chunks to produce (must be at least 1)
+        '''
+        if num_chunks < 1:
+            raise RuntimeError('You must have at least one chunk')
+        chunk_length = int(np.ceil(len(self.files) / num_chunks))
+        self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)]
+
+    def insert_bg_files_into_chunks(self, bgshift_function='pass'):
+        average_window = len(self.background_files)
+        for i, chunk in enumerate(self.chunked_files):
+            if i > 0 and bgshift_function != 'pass':
+                # If the bgshift_function is not pass then we need to find a new set of
+                # background images for the start of next chunk. These will be the last
+                # average_window number of files from the previous chunk.
+                # If bgshift_function is 'pass', then we should use the same background files for all chunks
+                # so there is no need to extend the list of background files here
+                self.background_files.extend(self.chunked_files[i-1][-average_window:])
+            # we have to loop backwards over bg_files because we are inserting into the top of the chunk
+            chunk = [chunk.insert(0, bg_file) for bg_file in reversed(self.background_files[-average_window:])]
+
+    def build_initial_background_files(self, average_window=0):
+        '''
+        Create a list of files to use for initializing the background in the first chunk
+        '''
+        self.background_files = []
+        for f in self.files[0:average_window]:
+            self.background_files.append(f)
+
+    def __len__(self):
+        return len(self.files)
+
+    def __iter__(self):
+        for filename in self.files:
+            yield filename

{pyopia-2.2.0 → pyopia-2.4.0}/pyopia/process.py

@@ -342,7 +342,7 @@ def measure_particles(imbw, max_particles=5000):
         raise RuntimeError('Too many particles. Refer to documentation on max_particles parameter in measure_particles()')
         # @todo handle situation when too many particles are found
 
-    region_properties = measure.regionprops(iml, cache=False)
+    region_properties = measure.regionprops(iml, cache=True)
 
     return region_properties
 

pyopia-2.2.0/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.2.0'