PyOPIA 2.6.0__tar.gz → 2.6.2__tar.gz

{pyopia-2.6.0 → pyopia-2.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyOPIA
-Version: 2.6.0
+Version: 2.6.2
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Home-page: https://github.com/sintef/pyopia
 Keywords: Ocean,Particles,Imaging,Measurement,Size distribution

pyopia-2.6.2/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.6.2'

{pyopia-2.6.0 → pyopia-2.6.2}/pyopia/cli.py

@@ -5,14 +5,17 @@ PyOPIA top-level code primarily for managing cmd line entry points
 import typer
 import toml
 import os
+import time
 import datetime
 import traceback
 import logging
-from rich.progress import track, Progress
+from rich.progress import Progress
 from rich.logging import RichHandler
+import rich.progress
 import pandas as pd
 import threading
 
+import pyopia
 import pyopia.background
 import pyopia.instrument.silcam
 import pyopia.instrument.holo
@@ -115,23 +118,27 @@ def generate_config(instrument: str, raw_files: str, model_path: str, outfolder:
 
 
 @app.command()
-def process(config_filename: str, num_chunks: int = 1):
+def process(config_filename: str, num_chunks: int = 1, strategy: str = 'block'):
     '''Run a PyOPIA processing pipeline based on given a config.toml
 
     Parameters
     ----------
     config_filename : str
-        config filename
+        Config filename
+
     numchunks : int, optional
-        split the dataset into chucks, and process in parallell, by default 1
+        Split the dataset into chucks, and process in parallell, by default 1
 
+    strategy : str, optional
+        Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block`
     '''
-    from pyopia.io import load_toml
-    from pyopia.pipeline import Pipeline
+    t1 = time.time()
 
     with Progress(transient=True) as progress:
+        progress.console.print(f"[blue]PYOPIA VERSION {pyopia.__version__}")
+
         progress.console.print("[blue]LOAD CONFIG")
-        pipeline_config = load_toml(config_filename)
+        pipeline_config = pyopia.io.load_toml(config_filename)
 
         setup_logging(pipeline_config)
         logger = logging.getLogger('rich')
@@ -140,11 +147,14 @@ def process(config_filename: str, num_chunks: int = 1):
         check_chunks(num_chunks, pipeline_config)
 
         progress.console.print("[blue]OBTAIN IMAGE LIST")
-        raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
         conf_corrbg = pipeline_config['steps'].get('correctbackground', dict())
         average_window = conf_corrbg.get('average_window', 0)
         bgshift_function = conf_corrbg.get('bgshift_function', 'pass')
-        raw_files.prepare_chunking(num_chunks, average_window, bgshift_function)
+        raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
+        raw_files.prepare_chunking(num_chunks, average_window, bgshift_function, strategy=strategy)
+
+        # Write the dataset list of images to a text file
+        raw_files.to_filelist_file('filelist.txt')
 
         progress.console.print('[blue]PREPARE FOLDERS')
         if 'output' not in pipeline_config['steps']:
@@ -163,26 +173,36 @@ def process(config_filename: str, num_chunks: int = 1):
         progress.console.print("[blue]INITIALISE PIPELINE")
 
     def process_file_list(file_list, c):
-        processing_pipeline = Pipeline(pipeline_config)
-        for filename in track(file_list, description=f'[blue]Processing progress (chunk {c})',
-                              disable=c != 0):
-            try:
-                logger.debug(f'Chunk {c} starting to process {filename}')
-                processing_pipeline.run(filename)
-            except Exception as e:
-                logger.warning('[red]An error occured in processing, ' +
-                               'skipping rest of pipeline and moving to next image.' +
-                               f'(chunk {c})')
-                logger.error(e)
-                logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+        processing_pipeline = pyopia.pipeline.Pipeline(pipeline_config)
+
+        with get_custom_progress_bar(f'[blue]Processing progress (chunk {c})', disable=c != 0) as pbar:
+            for filename in pbar.track(file_list, description=f'[blue]Processing progress (chunk {c})'):
+                try:
+                    logger.debug(f'Chunk {c} starting to process {filename}')
+                    processing_pipeline.run(filename)
+                except Exception as e:
+                    logger.warning('[red]An error occured in processing, ' +
+                                   'skipping rest of pipeline and moving to next image.' +
+                                   f'(chunk {c})')
+                    logger.error(e)
+                    logger.debug(''.join(traceback.format_tb(e.__traceback__)))
 
     # With one chunk we keep the non-threaded functionality to ensure backwards compatibility
+    job_list = []
     if num_chunks == 1:
         process_file_list(raw_files, 0)
     else:
         for c, chunk in enumerate(raw_files.chunked_files):
             job = threading.Thread(target=process_file_list, args=(chunk, c, ))
             job.start()
+            job_list.append(job)
+
+    # Calculate and print total processing time
+    # If we are using threads, make sure all jobs have finished
+    [job.join() for job in job_list]
+    time_total = pd.to_timedelta(time.time() - t1, 'seconds')
+    with Progress(transient=True) as progress:
+        progress.console.print(f"[blue]PROCESSING COMPLETED IN {time_total}")
 
 
 @app.command()
@@ -249,5 +269,21 @@ def check_chunks(chunks, pipeline_config):
         raise RuntimeError('Output mode must be set to "append = false" in "output" step when using more than one chunk')
 
 
+def get_custom_progress_bar(description, disable):
+    ''' Create a custom rich.progress.Progress object for displaying progress bars'''
+    progress = Progress(
+        rich.progress.TextColumn(description),
+        rich.progress.BarColumn(),
+        rich.progress.TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        rich.progress.MofNCompleteColumn(),
+        rich.progress.TextColumn("•"),
+        rich.progress.TimeElapsedColumn(),
+        rich.progress.TextColumn("•"),
+        rich.progress.TimeRemainingColumn(),
+        disable=disable
+    )
+    return progress
+
+
 if __name__ == "__main__":
     app()

{pyopia-2.6.0 → pyopia-2.6.2}/pyopia/pipeline.py

@@ -4,6 +4,7 @@ Module for managing the PyOpia processing pipeline
 Refer to the :class:`Pipeline` class documentation for examples of how to process datasets and images
 '''
 from typing import TypedDict
+import time
 import datetime
 import pandas as pd
 from operator import methodcaller
@@ -114,7 +115,10 @@ class Pipeline():
                 continue
 
             logger.info(f'Running pipeline step: {stepname}')
+            t1 = time.time()
             self.run_step(stepname)
+            t2 = time.time()
+            logger.debug(f'Running pipeline step {stepname} took {t2-t1:.3f} seconds')
 
             # Check for signal from this step that we should skip remaining pipeline for this image
             if self.data['skip_next_steps']:
@@ -369,25 +373,53 @@ class FilesToProcess:
         with open(path_to_filelist, 'w') as fh:
             [fh.writelines(L + '\n') for L in self.files]
 
-    def prepare_chunking(self, num_chunks, average_window, bgshift_function):
+        with open(path_to_filelist+'.chunks', 'w') as fh:
+            for i, chunk in enumerate(self.chunked_files):
+                fh.write(f'--- Chunk {i}, N = {len(chunk)} ---\n')
+                [fh.writelines(L + '\n') for L in chunk]
+
+    def prepare_chunking(self, num_chunks, average_window, bgshift_function, strategy='block'):
+        '''Chunk the file list and add initial background files to each chunk
+
+        Parameters
+        ----------
+        num_chunks : int
+            Number of chunks to produce (must be at least 1)
+        average_window : int
+            Number of images to use for background correction
+        bgshift_function : str
+            Background update strategy, either `pass` (static background) or `accurate`
+        strategy : str, optional
+            Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block`
+        '''
         if num_chunks > len(self.files) // 2:
             raise RuntimeError('Number of chunks exceeds more than half the number of files to process. Use less chunks.')
-        self.chunk_files(num_chunks)
+        self.chunk_files(num_chunks, strategy)
         self.build_initial_background_files(average_window=average_window)
         self.insert_bg_files_into_chunks(bgshift_function=bgshift_function)
 
-    def chunk_files(self, num_chunks: int):
+    def chunk_files(self, num_chunks: int, strategy: str = 'block'):
         '''Chunk the file list and create FilesToProcess.chunked_files
 
         Parameters
         ----------
         num_chunks : int
             number of chunks to produce (must be at least 1)
+        strategy : str, optional
+            Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block`
         '''
         if num_chunks < 1:
             raise RuntimeError('You must have at least one chunk')
         chunk_length = int(np.ceil(len(self.files) / num_chunks))
-        self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)]
+        if strategy == 'block':
+            logging.debug('Chunking file list with strategy: block')
+            self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)]
+        elif strategy == 'interleave':
+            logging.debug('Chunking file list with strategy: interleave')
+            self.chunked_files = [self.files[i::num_chunks] for i in range(0, num_chunks)]
+        else:
+            logging.debug(f'Invalid chunking strategy: {strategy}')
+            raise RuntimeError(f'Unknown strategy: {strategy}. Should be either block or interleave')
 
     def insert_bg_files_into_chunks(self, bgshift_function='pass'):
         average_window = len(self.background_files)

{pyopia-2.6.0 → pyopia-2.6.2}/pyopia/process.py

@@ -623,7 +623,7 @@ class CalculateImageStats():
 
         # Add image "global" statistics, separate from the particle stats above (stats)
         image_saturation = np.nan if stats.empty else stats['saturation'].values[0]
-        data['image_stats'].loc[data['timestamp'], 'filename'] = getattr(data, 'filename', '')
+        data['image_stats'].loc[data['timestamp'], 'filename'] = data.get('filename', '')
         data['image_stats'].loc[data['timestamp'], 'particle_count'] = int(stats.shape[0])
         data['image_stats'].loc[data['timestamp'], 'saturation'] = image_saturation
 

pyopia-2.6.0/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.6.0'