PyOPIA 2.6.1__tar.gz → 2.7.0__tar.gz

{pyopia-2.6.1 → pyopia-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyOPIA
-Version: 2.6.1
+Version: 2.7.0
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Home-page: https://github.com/sintef/pyopia
 Keywords: Ocean,Particles,Imaging,Measurement,Size distribution

pyopia-2.7.0/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.7.0'

{pyopia-2.6.1 → pyopia-2.7.0}/pyopia/background.py

@@ -95,9 +95,11 @@ def shift_bgstack_fast(bgstack, imbg, imnew):
     return bgstack, imbg
 
 
-def correct_im_accurate(imbg, imraw):
+def correct_im_accurate(imbg, imraw, divide_bg=False):
     '''
-    Corrects raw image by subtracting the background and scaling the output
+    Corrects raw image by subtracting or dividing the background and scaling the output
+
+    For dividing method see: https://doi.org/10.1016/j.marpolbul.2016.11.063)
 
     There is a small chance of clipping of imc in both crushed blacks and blown
     highlights if the background or raw images are very poorly obtained
@@ -108,6 +110,9 @@ def correct_im_accurate(imbg, imraw):
         background averaged image
     imraw : float64
         raw image
+    divide_bg : (bool, optional)
+        If True, the correction will be performed by dividing the raw image by the background
+        Default to False
 
     Returns
     -------
@@ -115,10 +120,16 @@ def correct_im_accurate(imbg, imraw):
         corrected image, same type as input
     '''
 
-    im_corrected = imraw - imbg
-    im_corrected += (1 / 2 - np.percentile(im_corrected, 50))
-
-    im_corrected += 1 - im_corrected.max()
+    if divide_bg:
+        imbg = np.clip(imbg, a_min=1/255, a_max=None)   # Clipping the zero_value pixels
+        im_corrected = imraw / imbg
+        im_corrected += (1 / 2 - np.percentile(im_corrected, 50))
+        im_corrected -= im_corrected.min()   # Shift the negative values to zero
+        im_corrected = np.clip(im_corrected, a_min=0, a_max=1)
+    else:
+        im_corrected = imraw - imbg
+        im_corrected += (1 / 2 - np.percentile(im_corrected, 50))
+        im_corrected += 1 - im_corrected.max()   # Shift the positive values exceeding unity to one
 
     return im_corrected
 
@@ -207,7 +218,7 @@ class CorrectBackgroundAccurate():
     ----------
     bgshift_function : (string, optional)
         Function used to shift the background. Defaults to passing (i.e. static background)
-        Available options are 'accurate', 'fast', or 'pass' to apply a statick background correction:
+        Available options are 'accurate', 'fast', or 'pass' to apply a static background correction:
 
         :func:`pyopia.background.shift_bgstack_accurate`
 
@@ -220,6 +231,10 @@ class CorrectBackgroundAccurate():
         The key in Pipeline.data of the image to be background corrected.
         Defaults to 'imraw'
 
+    divide_bg : (bool)
+        If True, it performs background correction by dividing the raw image by the background.
+        Default to False.
+
     Returns
     -------
     data : :class:`pyopia.pipeline.Data`
@@ -256,10 +271,11 @@ class CorrectBackgroundAccurate():
     Then you could use :class:`pyopia.pipeline.CorrectBackgroundNone` if you need to instead.
     '''
 
-    def __init__(self, bgshift_function='pass', average_window=1, image_source='imraw'):
+    def __init__(self, bgshift_function='pass', average_window=1, image_source='imraw', divide_bg=False):
         self.bgshift_function = bgshift_function
         self.average_window = average_window
         self.image_source = image_source
+        self.divide_bg = divide_bg
 
     def _build_background_step(self, data):
         '''Add one layer to the background stack from the raw image in data pipeline, and update the background image.'''
@@ -284,7 +300,7 @@ class CorrectBackgroundAccurate():
             data['skip_next_steps'] = True
             return data
 
-        data['im_corrected'] = correct_im_accurate(data['imbg'], data[self.image_source])
+        data['im_corrected'] = correct_im_accurate(data['imbg'], data[self.image_source], divide_bg=self.divide_bg)
 
         match self.bgshift_function:
             case 'pass':

{pyopia-2.6.1 → pyopia-2.7.0}/pyopia/cli.py

@@ -15,6 +15,7 @@ import rich.progress
 import pandas as pd
 import threading
 
+import pyopia
 import pyopia.background
 import pyopia.instrument.silcam
 import pyopia.instrument.holo
@@ -117,25 +118,27 @@ def generate_config(instrument: str, raw_files: str, model_path: str, outfolder:
 
 
 @app.command()
-def process(config_filename: str, num_chunks: int = 1):
+def process(config_filename: str, num_chunks: int = 1, strategy: str = 'block'):
     '''Run a PyOPIA processing pipeline based on given a config.toml
 
     Parameters
     ----------
     config_filename : str
-        config filename
+        Config filename
+
     numchunks : int, optional
-        split the dataset into chucks, and process in parallell, by default 1
+        Split the dataset into chucks, and process in parallell, by default 1
 
+    strategy : str, optional
+        Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block`
     '''
-    from pyopia.io import load_toml
-    from pyopia.pipeline import Pipeline
-
     t1 = time.time()
 
     with Progress(transient=True) as progress:
+        progress.console.print(f"[blue]PYOPIA VERSION {pyopia.__version__}")
+
         progress.console.print("[blue]LOAD CONFIG")
-        pipeline_config = load_toml(config_filename)
+        pipeline_config = pyopia.io.load_toml(config_filename)
 
         setup_logging(pipeline_config)
         logger = logging.getLogger('rich')
@@ -144,11 +147,14 @@ def process(config_filename: str, num_chunks: int = 1):
         check_chunks(num_chunks, pipeline_config)
 
         progress.console.print("[blue]OBTAIN IMAGE LIST")
-        raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
         conf_corrbg = pipeline_config['steps'].get('correctbackground', dict())
         average_window = conf_corrbg.get('average_window', 0)
         bgshift_function = conf_corrbg.get('bgshift_function', 'pass')
-        raw_files.prepare_chunking(num_chunks, average_window, bgshift_function)
+        raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
+        raw_files.prepare_chunking(num_chunks, average_window, bgshift_function, strategy=strategy)
+
+        # Write the dataset list of images to a text file
+        raw_files.to_filelist_file('filelist.txt')
 
         progress.console.print('[blue]PREPARE FOLDERS')
         if 'output' not in pipeline_config['steps']:
@@ -167,7 +173,7 @@ def process(config_filename: str, num_chunks: int = 1):
         progress.console.print("[blue]INITIALISE PIPELINE")
 
     def process_file_list(file_list, c):
-        processing_pipeline = Pipeline(pipeline_config)
+        processing_pipeline = pyopia.pipeline.Pipeline(pipeline_config)
 
         with get_custom_progress_bar(f'[blue]Processing progress (chunk {c})', disable=c != 0) as pbar:
             for filename in pbar.track(file_list, description=f'[blue]Processing progress (chunk {c})'):

{pyopia-2.6.1 → pyopia-2.7.0}/pyopia/pipeline.py

@@ -373,25 +373,53 @@ class FilesToProcess:
         with open(path_to_filelist, 'w') as fh:
             [fh.writelines(L + '\n') for L in self.files]
 
-    def prepare_chunking(self, num_chunks, average_window, bgshift_function):
+        with open(path_to_filelist+'.chunks', 'w') as fh:
+            for i, chunk in enumerate(self.chunked_files):
+                fh.write(f'--- Chunk {i}, N = {len(chunk)} ---\n')
+                [fh.writelines(L + '\n') for L in chunk]
+
+    def prepare_chunking(self, num_chunks, average_window, bgshift_function, strategy='block'):
+        '''Chunk the file list and add initial background files to each chunk
+
+        Parameters
+        ----------
+        num_chunks : int
+            Number of chunks to produce (must be at least 1)
+        average_window : int
+            Number of images to use for background correction
+        bgshift_function : str
+            Background update strategy, either `pass` (static background) or `accurate`
+        strategy : str, optional
+            Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block`
+        '''
         if num_chunks > len(self.files) // 2:
             raise RuntimeError('Number of chunks exceeds more than half the number of files to process. Use less chunks.')
-        self.chunk_files(num_chunks)
+        self.chunk_files(num_chunks, strategy)
         self.build_initial_background_files(average_window=average_window)
         self.insert_bg_files_into_chunks(bgshift_function=bgshift_function)
 
-    def chunk_files(self, num_chunks: int):
+    def chunk_files(self, num_chunks: int, strategy: str = 'block'):
         '''Chunk the file list and create FilesToProcess.chunked_files
 
         Parameters
         ----------
         num_chunks : int
             number of chunks to produce (must be at least 1)
+        strategy : str, optional
+            Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block`
         '''
         if num_chunks < 1:
             raise RuntimeError('You must have at least one chunk')
         chunk_length = int(np.ceil(len(self.files) / num_chunks))
-        self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)]
+        if strategy == 'block':
+            logging.debug('Chunking file list with strategy: block')
+            self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)]
+        elif strategy == 'interleave':
+            logging.debug('Chunking file list with strategy: interleave')
+            self.chunked_files = [self.files[i::num_chunks] for i in range(0, num_chunks)]
+        else:
+            logging.debug(f'Invalid chunking strategy: {strategy}')
+            raise RuntimeError(f'Unknown strategy: {strategy}. Should be either block or interleave')
 
     def insert_bg_files_into_chunks(self, bgshift_function='pass'):
         average_window = len(self.background_files)

{pyopia-2.6.1 → pyopia-2.7.0}/pyopia/process.py

@@ -623,7 +623,7 @@ class CalculateImageStats():
 
         # Add image "global" statistics, separate from the particle stats above (stats)
         image_saturation = np.nan if stats.empty else stats['saturation'].values[0]
-        data['image_stats'].loc[data['timestamp'], 'filename'] = getattr(data, 'filename', '')
+        data['image_stats'].loc[data['timestamp'], 'filename'] = data.get('filename', '')
         data['image_stats'].loc[data['timestamp'], 'particle_count'] = int(stats.shape[0])
         data['image_stats'].loc[data['timestamp'], 'saturation'] = image_saturation
 

{pyopia-2.6.1 → pyopia-2.7.0}/pyopia/statistics.py

@@ -534,7 +534,7 @@ def get_j(dias, number_distribution):
 
     # use polyfit to obtain the slope of the ditriubtion in log-space (which is
     # assumed near-linear in most parts of the ocean)
-    p = np.polyfit(np.log(dias[ind]), np.log(number_distribution[ind]), 1)
+    p = np.polyfit(np.log(dias[ind]), np.log(number_distribution[ind], where=number_distribution[ind] > 0), 1)
     junge_slope = p[0]
     return junge_slope
 

pyopia-2.6.1/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.6.1'