PyOPIA 2.6.0__tar.gz → 2.6.1__tar.gz

{pyopia-2.6.0 → pyopia-2.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyOPIA
-Version: 2.6.0
+Version: 2.6.1
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Home-page: https://github.com/sintef/pyopia
 Keywords: Ocean,Particles,Imaging,Measurement,Size distribution

pyopia-2.6.1/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.6.1'

{pyopia-2.6.0 → pyopia-2.6.1}/pyopia/cli.py

@@ -5,11 +5,13 @@ PyOPIA top-level code primarily for managing cmd line entry points
 import typer
 import toml
 import os
+import time
 import datetime
 import traceback
 import logging
-from rich.progress import track, Progress
+from rich.progress import Progress
 from rich.logging import RichHandler
+import rich.progress
 import pandas as pd
 import threading
 
@@ -129,6 +131,8 @@ def process(config_filename: str, num_chunks: int = 1):
     from pyopia.io import load_toml
     from pyopia.pipeline import Pipeline
 
+    t1 = time.time()
+
     with Progress(transient=True) as progress:
         progress.console.print("[blue]LOAD CONFIG")
         pipeline_config = load_toml(config_filename)
@@ -164,25 +168,35 @@ def process(config_filename: str, num_chunks: int = 1):
 
     def process_file_list(file_list, c):
         processing_pipeline = Pipeline(pipeline_config)
-        for filename in track(file_list, description=f'[blue]Processing progress (chunk {c})',
-                              disable=c != 0):
-            try:
-                logger.debug(f'Chunk {c} starting to process {filename}')
-                processing_pipeline.run(filename)
-            except Exception as e:
-                logger.warning('[red]An error occured in processing, ' +
-                               'skipping rest of pipeline and moving to next image.' +
-                               f'(chunk {c})')
-                logger.error(e)
-                logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+
+        with get_custom_progress_bar(f'[blue]Processing progress (chunk {c})', disable=c != 0) as pbar:
+            for filename in pbar.track(file_list, description=f'[blue]Processing progress (chunk {c})'):
+                try:
+                    logger.debug(f'Chunk {c} starting to process {filename}')
+                    processing_pipeline.run(filename)
+                except Exception as e:
+                    logger.warning('[red]An error occured in processing, ' +
+                                   'skipping rest of pipeline and moving to next image.' +
+                                   f'(chunk {c})')
+                    logger.error(e)
+                    logger.debug(''.join(traceback.format_tb(e.__traceback__)))
 
     # With one chunk we keep the non-threaded functionality to ensure backwards compatibility
+    job_list = []
     if num_chunks == 1:
         process_file_list(raw_files, 0)
     else:
         for c, chunk in enumerate(raw_files.chunked_files):
             job = threading.Thread(target=process_file_list, args=(chunk, c, ))
             job.start()
+            job_list.append(job)
+
+    # Calculate and print total processing time
+    # If we are using threads, make sure all jobs have finished
+    [job.join() for job in job_list]
+    time_total = pd.to_timedelta(time.time() - t1, 'seconds')
+    with Progress(transient=True) as progress:
+        progress.console.print(f"[blue]PROCESSING COMPLETED IN {time_total}")
 
 
 @app.command()
@@ -249,5 +263,21 @@ def check_chunks(chunks, pipeline_config):
         raise RuntimeError('Output mode must be set to "append = false" in "output" step when using more than one chunk')
 
 
+def get_custom_progress_bar(description, disable):
+    ''' Create a custom rich.progress.Progress object for displaying progress bars'''
+    progress = Progress(
+        rich.progress.TextColumn(description),
+        rich.progress.BarColumn(),
+        rich.progress.TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        rich.progress.MofNCompleteColumn(),
+        rich.progress.TextColumn("•"),
+        rich.progress.TimeElapsedColumn(),
+        rich.progress.TextColumn("•"),
+        rich.progress.TimeRemainingColumn(),
+        disable=disable
+    )
+    return progress
+
+
 if __name__ == "__main__":
     app()

{pyopia-2.6.0 → pyopia-2.6.1}/pyopia/pipeline.py

@@ -4,6 +4,7 @@ Module for managing the PyOpia processing pipeline
 Refer to the :class:`Pipeline` class documentation for examples of how to process datasets and images
 '''
 from typing import TypedDict
+import time
 import datetime
 import pandas as pd
 from operator import methodcaller
@@ -114,7 +115,10 @@ class Pipeline():
                 continue
 
             logger.info(f'Running pipeline step: {stepname}')
+            t1 = time.time()
             self.run_step(stepname)
+            t2 = time.time()
+            logger.debug(f'Running pipeline step {stepname} took {t2-t1:.3f} seconds')
 
             # Check for signal from this step that we should skip remaining pipeline for this image
             if self.data['skip_next_steps']:

pyopia-2.6.0/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.6.0'