PyOPIA 2.3.0__tar.gz → 2.4.0__tar.gz

{pyopia-2.3.0 → pyopia-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyOPIA
-Version: 2.3.0
+Version: 2.4.0
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Home-page: https://github.com/sintef/pyopia
 Keywords: Ocean,Particles,Imaging,Measurement,Size distribution

pyopia-2.4.0/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.4.0'

{pyopia-2.3.0 → pyopia-2.4.0}/pyopia/cli.py

@@ -4,13 +4,14 @@ PyOPIA top-level code primarily for managing cmd line entry points
 
 import typer
 import toml
-from glob import glob
 import os
 import datetime
 import traceback
 import logging
 from rich.progress import track, Progress
+from rich.logging import RichHandler
 import pandas as pd
+import threading
 
 import pyopia.background
 import pyopia.instrument.silcam
@@ -105,23 +106,35 @@ def generate_config(instrument: str, raw_files: str, model_path: str, outfolder:
 
 
 @app.command()
-def process(config_filename: str):
+def process(config_filename: str, num_chunks: int = 1):
     '''Run a PyOPIA processing pipeline based on given a config.toml
+
+    Parameters
+    ----------
+    config_filename : str
+        config filename
+    numchunks : int, optional
+        split the dataset into chucks, and process in parallell, by default 1
+
     '''
     from pyopia.io import load_toml
     from pyopia.pipeline import Pipeline
 
-    logger = logging.getLogger()
-
     with Progress(transient=True) as progress:
         progress.console.print("[blue]LOAD CONFIG")
         pipeline_config = load_toml(config_filename)
 
         setup_logging(pipeline_config)
+        logger = logging.getLogger('rich')
+        logger.info(f'PyOPIA process started {pd.Timestamp.now()}')
+
+        check_chunks(num_chunks, pipeline_config)
 
         progress.console.print("[blue]OBTAIN FILE LIST")
-        files = sorted(glob(pipeline_config['general']['raw_files']))
-        nfiles = len(files)
+        raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
+        average_window = pipeline_config['steps']['correctbackground'].get('average_window', 0)
+        bgshift_function = pipeline_config['steps']['correctbackground'].get('bgshift_function', 'pass')
+        raw_files.prepare_chunking(num_chunks, average_window, bgshift_function)
 
         progress.console.print('[blue]PREPARE FOLDERS')
         if 'output' not in pipeline_config['steps']:
@@ -134,20 +147,32 @@ def process(config_filename: str):
         if os.path.isfile(output_datafile + '-STATS.nc'):
             dt_now = datetime.datetime.now().strftime('D%Y%m%dT%H%M%S')
             newname = output_datafile + '-conflict-' + str(dt_now) + '-STATS.nc'
-            progress.console.print('[red]Renaming conflicting file to: ' +
-                                   newname)
+            logger.warning(f'Renaming conflicting file to: {newname}')
             os.rename(output_datafile + '-STATS.nc', newname)
 
         progress.console.print("[blue]INITIALISE PIPELINE")
-        processing_pipeline = Pipeline(pipeline_config)
 
-    for filename in track(files, description=f'[blue]Processing progress through {nfiles} files:'):
-        try:
-            processing_pipeline.run(filename)
-        except Exception as e:
-            progress.console.print("[red]An error occured in processing, skipping rest of pipeline and moving to next image.")
-            logger.error(e)
-            logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+    def process_file_list(file_list, c):
+        processing_pipeline = Pipeline(pipeline_config)
+        for filename in track(file_list, description=f'[blue]Processing progress (chunk {c})',
+                              disable=c != 0):
+            try:
+                logger.debug(f'Chunk {c} starting to process {filename}')
+                processing_pipeline.run(filename)
+            except Exception as e:
+                logger.warning('[red]An error occured in processing, ' +
+                               'skipping rest of pipeline and moving to next image.' +
+                               f'(chunk {c})')
+                logger.error(e)
+                logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+
+    # With one chunk we keep the non-threaded functionality to ensure backwards compatibility
+    if num_chunks == 1:
+        process_file_list(raw_files, 0)
+    else:
+        for c, chunk in enumerate(raw_files.chunked_files):
+            job = threading.Thread(target=process_file_list, args=(chunk, c, ))
+            job.start()
 
 
 @app.command()
@@ -179,15 +204,25 @@ def setup_logging(pipeline_config):
     log_file = pipeline_config['general'].get('log_file', None)
     log_level_name = pipeline_config['general'].get('log_level', 'INFO')
     log_level = getattr(logging, log_level_name)
-    print(log_level_name, log_level, log_file)
+
+    # Either log to file (silent console) or to console with Rich
+    if log_file is None:
+        handlers = [RichHandler(show_time=True, show_level=False)]
+    else:
+        handlers = [logging.FileHandler(log_file, mode='a')]
 
     # Configure logger
-    log_format = '%(asctime)s %(levelname)s [%(module)s.%(funcName)s] %(message)s'
-    logging.basicConfig(level=log_level, format=log_format, filename=log_file,
-                        datefmt='%Y-%m-%d %H:%M:%S')
+    log_format = '%(asctime)s %(levelname)s %(threadName)s [%(module)s.%(funcName)s] %(message)s'
+    logging.basicConfig(level=log_level, datefmt='%Y-%m-%d %H:%M:%S', format=log_format, handlers=handlers)
+
+
+def check_chunks(chunks, pipeline_config):
+    if chunks < 1:
+        raise RuntimeError('You must have at least 1 chunk')
 
-    logger = logging.getLogger()
-    logger.info(f'PyOPIA process started {pd.Timestamp.now()}')
+    append_enabled = pipeline_config['steps']['output'].get('append', True)
+    if chunks > 1 and append_enabled:
+        raise RuntimeError('Output mode must be set to "append = false" in "output" step when using more than one chunk')
 
 
 if __name__ == "__main__":

{pyopia-2.3.0 → pyopia-2.4.0}/pyopia/pipeline.py

@@ -10,6 +10,8 @@ from operator import methodcaller
 import sys
 from pyopia.io import steps_from_xstats as steps_from_xstats # noqa: E(F401)
 import logging
+from glob import glob
+import numpy as np
 
 logger = logging.getLogger()
 
@@ -322,3 +324,88 @@ def build_steps(toml_steps):
         steps[step_name] = build_repr(toml_steps, step_name)
 
     return steps
+
+
+class FilesToProcess:
+    def __init__(self, glob_pattern=None):
+        '''Build file list from glob pattern if specified.
+           Create FilesToProcess.chunked_files is chunks specified
+           File list from glob will be sorted.
+
+        Parameters
+        ----------
+        glob_pattern : str, optional
+            Glob pattern, by default None
+        '''
+        self.files = None
+        self.background_files = []
+        self.chunked_files = []
+        if glob_pattern is not None:
+            self.files = sorted(glob(glob_pattern))
+
+    def from_filelist_file(self, path_to_filelist):
+        '''
+        Initialize explicit list of files to process from a text file.
+        The text file should contain one path to an image per line, which should be processed in order.
+        '''
+        with open(path_to_filelist, 'r') as fh:
+            self.files = list(fh.readlines())
+
+    def to_filelist_file(self, path_to_filelist):
+        '''Write file list to a txt file
+
+        Parameters
+        ----------
+        path_to_filelist : str
+            Path to txt file to write
+        '''
+        with open(path_to_filelist, 'w') as fh:
+            [fh.writelines(L + '\n') for L in self.files]
+
+    def prepare_chunking(self, num_chunks, average_window, bgshift_function):
+        if num_chunks > len(self.files) // 2:
+            raise RuntimeError('Number of chunks exceeds more than half the number of files to process. Use less chunks.')
+        self.chunk_files(num_chunks)
+        self.build_initial_background_files(average_window=average_window)
+        self.insert_bg_files_into_chunks(bgshift_function=bgshift_function)
+
+    def chunk_files(self, num_chunks: int):
+        '''Chunk the file list and create FilesToProcess.chunked_files
+
+        Parameters
+        ----------
+        chunks : int
+            number of chunks to produce (must be at least 1)
+        '''
+        if num_chunks < 1:
+            raise RuntimeError('You must have at least one chunk')
+        chunk_length = int(np.ceil(len(self.files) / num_chunks))
+        self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)]
+
+    def insert_bg_files_into_chunks(self, bgshift_function='pass'):
+        average_window = len(self.background_files)
+        for i, chunk in enumerate(self.chunked_files):
+            if i > 0 and bgshift_function != 'pass':
+                # If the bgshift_function is not pass then we need to find a new set of
+                # background images for the start of next chunk. These will be the last
+                # average_window number of files from the previous chunk.
+                # If bgshift_function is 'pass', then we should use the same background files for all chunks
+                # so there is no need to extend the list of background files here
+                self.background_files.extend(self.chunked_files[i-1][-average_window:])
+            # we have to loop backwards over bg_files because we are inserting into the top of the chunk
+            chunk = [chunk.insert(0, bg_file) for bg_file in reversed(self.background_files[-average_window:])]
+
+    def build_initial_background_files(self, average_window=0):
+        '''
+        Create a list of files to use for initializing the background in the first chunk
+        '''
+        self.background_files = []
+        for f in self.files[0:average_window]:
+            self.background_files.append(f)
+
+    def __len__(self):
+        return len(self.files)
+
+    def __iter__(self):
+        for filename in self.files:
+            yield filename

{pyopia-2.3.0 → pyopia-2.4.0}/pyopia/process.py

@@ -342,7 +342,7 @@ def measure_particles(imbw, max_particles=5000):
         raise RuntimeError('Too many particles. Refer to documentation on max_particles parameter in measure_particles()')
         # @todo handle situation when too many particles are found
 
-    region_properties = measure.regionprops(iml, cache=False)
+    region_properties = measure.regionprops(iml, cache=True)
 
     return region_properties
 

pyopia-2.3.0/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.3.0'