PyOPIA 2.5.13__tar.gz → 2.6.1__tar.gz

{pyopia-2.5.13 → pyopia-2.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyOPIA
-Version: 2.5.13
+Version: 2.6.1
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Home-page: https://github.com/sintef/pyopia
 Keywords: Ocean,Particles,Imaging,Measurement,Size distribution

pyopia-2.6.1/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.6.1'

{pyopia-2.5.13 → pyopia-2.6.1}/pyopia/cli.py

@@ -5,11 +5,13 @@ PyOPIA top-level code primarily for managing cmd line entry points
 import typer
 import toml
 import os
+import time
 import datetime
 import traceback
 import logging
-from rich.progress import track, Progress
+from rich.progress import Progress
 from rich.logging import RichHandler
+import rich.progress
 import pandas as pd
 import threading
 
@@ -129,6 +131,8 @@ def process(config_filename: str, num_chunks: int = 1):
     from pyopia.io import load_toml
     from pyopia.pipeline import Pipeline
 
+    t1 = time.time()
+
     with Progress(transient=True) as progress:
         progress.console.print("[blue]LOAD CONFIG")
         pipeline_config = load_toml(config_filename)
@@ -139,7 +143,7 @@ def process(config_filename: str, num_chunks: int = 1):
 
         check_chunks(num_chunks, pipeline_config)
 
-        progress.console.print("[blue]OBTAIN FILE LIST")
+        progress.console.print("[blue]OBTAIN IMAGE LIST")
         raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
         conf_corrbg = pipeline_config['steps'].get('correctbackground', dict())
         average_window = conf_corrbg.get('average_window', 0)
@@ -164,25 +168,35 @@ def process(config_filename: str, num_chunks: int = 1):
 
     def process_file_list(file_list, c):
         processing_pipeline = Pipeline(pipeline_config)
-        for filename in track(file_list, description=f'[blue]Processing progress (chunk {c})',
-                              disable=c != 0):
-            try:
-                logger.debug(f'Chunk {c} starting to process {filename}')
-                processing_pipeline.run(filename)
-            except Exception as e:
-                logger.warning('[red]An error occured in processing, ' +
-                               'skipping rest of pipeline and moving to next image.' +
-                               f'(chunk {c})')
-                logger.error(e)
-                logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+
+        with get_custom_progress_bar(f'[blue]Processing progress (chunk {c})', disable=c != 0) as pbar:
+            for filename in pbar.track(file_list, description=f'[blue]Processing progress (chunk {c})'):
+                try:
+                    logger.debug(f'Chunk {c} starting to process {filename}')
+                    processing_pipeline.run(filename)
+                except Exception as e:
+                    logger.warning('[red]An error occured in processing, ' +
+                                   'skipping rest of pipeline and moving to next image.' +
+                                   f'(chunk {c})')
+                    logger.error(e)
+                    logger.debug(''.join(traceback.format_tb(e.__traceback__)))
 
     # With one chunk we keep the non-threaded functionality to ensure backwards compatibility
+    job_list = []
     if num_chunks == 1:
         process_file_list(raw_files, 0)
     else:
         for c, chunk in enumerate(raw_files.chunked_files):
             job = threading.Thread(target=process_file_list, args=(chunk, c, ))
             job.start()
+            job_list.append(job)
+
+    # Calculate and print total processing time
+    # If we are using threads, make sure all jobs have finished
+    [job.join() for job in job_list]
+    time_total = pd.to_timedelta(time.time() - t1, 'seconds')
+    with Progress(transient=True) as progress:
+        progress.console.print(f"[blue]PROCESSING COMPLETED IN {time_total}")
 
 
 @app.command()
@@ -249,5 +263,21 @@ def check_chunks(chunks, pipeline_config):
         raise RuntimeError('Output mode must be set to "append = false" in "output" step when using more than one chunk')
 
 
+def get_custom_progress_bar(description, disable):
+    ''' Create a custom rich.progress.Progress object for displaying progress bars'''
+    progress = Progress(
+        rich.progress.TextColumn(description),
+        rich.progress.BarColumn(),
+        rich.progress.TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        rich.progress.MofNCompleteColumn(),
+        rich.progress.TextColumn("•"),
+        rich.progress.TimeElapsedColumn(),
+        rich.progress.TextColumn("•"),
+        rich.progress.TimeRemainingColumn(),
+        disable=disable
+    )
+    return progress
+
+
 if __name__ == "__main__":
     app()

{pyopia-2.5.13 → pyopia-2.6.1}/pyopia/pipeline.py

@@ -4,6 +4,7 @@ Module for managing the PyOpia processing pipeline
 Refer to the :class:`Pipeline` class documentation for examples of how to process datasets and images
 '''
 from typing import TypedDict
+import time
 import datetime
 import pandas as pd
 from operator import methodcaller
@@ -114,7 +115,10 @@ class Pipeline():
                 continue
 
             logger.info(f'Running pipeline step: {stepname}')
+            t1 = time.time()
             self.run_step(stepname)
+            t2 = time.time()
+            logger.debug(f'Running pipeline step {stepname} took {t2-t1:.3f} seconds')
 
             # Check for signal from this step that we should skip remaining pipeline for this image
             if self.data['skip_next_steps']:
@@ -331,26 +335,32 @@ class FilesToProcess:
     '''Build file list from glob pattern if specified.
     Create FilesToProcess.chunked_files is chunks specified
     File list from glob will be sorted.
+    If a filelist file is specified, load the list from there without sorting.
 
     Parameters
     ----------
     glob_pattern : str, optional
-        Glob pattern, by default None
+        Glob pattern, by default None. If it ends with .txt, interpret as a filelist file.
     '''
     def __init__(self, glob_pattern=None):
         self.files = None
         self.background_files = []
         self.chunked_files = []
         if glob_pattern is not None:
-            self.files = sorted(glob(glob_pattern))
+            # If a .txt file is specified, this indicates we should get the filelist from there
+            if glob_pattern.endswith('.txt'):
+                self.from_filelist_file(glob_pattern)
+            else:
+                self.files = sorted(glob(glob_pattern))
 
     def from_filelist_file(self, path_to_filelist):
         '''
         Initialize explicit list of files to process from a text file.
         The text file should contain one path to an image per line, which should be processed in order.
         '''
+        logger.info(f'Loading explicit image file list from file: {path_to_filelist}')
         with open(path_to_filelist, 'r') as fh:
-            self.files = list(fh.readlines())
+            self.files = [line.rstrip() for line in fh.readlines()]
 
     def to_filelist_file(self, path_to_filelist):
         '''Write file list to a txt file

pyopia-2.5.13/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.5.13'