PyOPIA 2.6.1__tar.gz → 2.6.2__tar.gz

{pyopia-2.6.1 → pyopia-2.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyOPIA
-Version: 2.6.1
+Version: 2.6.2
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Home-page: https://github.com/sintef/pyopia
 Keywords: Ocean,Particles,Imaging,Measurement,Size distribution

pyopia-2.6.2/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.6.2'

{pyopia-2.6.1 → pyopia-2.6.2}/pyopia/cli.py

@@ -15,6 +15,7 @@ import rich.progress
 import pandas as pd
 import threading
 
+import pyopia
 import pyopia.background
 import pyopia.instrument.silcam
 import pyopia.instrument.holo
@@ -117,25 +118,27 @@ def generate_config(instrument: str, raw_files: str, model_path: str, outfolder:
 
 
 @app.command()
-def process(config_filename: str, num_chunks: int = 1):
+def process(config_filename: str, num_chunks: int = 1, strategy: str = 'block'):
     '''Run a PyOPIA processing pipeline based on given a config.toml
 
     Parameters
     ----------
     config_filename : str
-        config filename
+        Config filename
+
     numchunks : int, optional
-        split the dataset into chucks, and process in parallell, by default 1
+        Split the dataset into chucks, and process in parallell, by default 1
 
+    strategy : str, optional
+        Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block`
     '''
-    from pyopia.io import load_toml
-    from pyopia.pipeline import Pipeline
-
     t1 = time.time()
 
     with Progress(transient=True) as progress:
+        progress.console.print(f"[blue]PYOPIA VERSION {pyopia.__version__}")
+
         progress.console.print("[blue]LOAD CONFIG")
-        pipeline_config = load_toml(config_filename)
+        pipeline_config = pyopia.io.load_toml(config_filename)
 
         setup_logging(pipeline_config)
         logger = logging.getLogger('rich')
@@ -144,11 +147,14 @@ def process(config_filename: str, num_chunks: int = 1):
         check_chunks(num_chunks, pipeline_config)
 
         progress.console.print("[blue]OBTAIN IMAGE LIST")
-        raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
         conf_corrbg = pipeline_config['steps'].get('correctbackground', dict())
         average_window = conf_corrbg.get('average_window', 0)
         bgshift_function = conf_corrbg.get('bgshift_function', 'pass')
-        raw_files.prepare_chunking(num_chunks, average_window, bgshift_function)
+        raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
+        raw_files.prepare_chunking(num_chunks, average_window, bgshift_function, strategy=strategy)
+
+        # Write the dataset list of images to a text file
+        raw_files.to_filelist_file('filelist.txt')
 
         progress.console.print('[blue]PREPARE FOLDERS')
         if 'output' not in pipeline_config['steps']:
@@ -167,7 +173,7 @@ def process(config_filename: str, num_chunks: int = 1):
         progress.console.print("[blue]INITIALISE PIPELINE")
 
     def process_file_list(file_list, c):
-        processing_pipeline = Pipeline(pipeline_config)
+        processing_pipeline = pyopia.pipeline.Pipeline(pipeline_config)
 
         with get_custom_progress_bar(f'[blue]Processing progress (chunk {c})', disable=c != 0) as pbar:
             for filename in pbar.track(file_list, description=f'[blue]Processing progress (chunk {c})'):

{pyopia-2.6.1 → pyopia-2.6.2}/pyopia/pipeline.py

@@ -373,25 +373,53 @@ class FilesToProcess:
         with open(path_to_filelist, 'w') as fh:
             [fh.writelines(L + '\n') for L in self.files]
 
-    def prepare_chunking(self, num_chunks, average_window, bgshift_function):
+        with open(path_to_filelist+'.chunks', 'w') as fh:
+            for i, chunk in enumerate(self.chunked_files):
+                fh.write(f'--- Chunk {i}, N = {len(chunk)} ---\n')
+                [fh.writelines(L + '\n') for L in chunk]
+
+    def prepare_chunking(self, num_chunks, average_window, bgshift_function, strategy='block'):
+        '''Chunk the file list and add initial background files to each chunk
+
+        Parameters
+        ----------
+        num_chunks : int
+            Number of chunks to produce (must be at least 1)
+        average_window : int
+            Number of images to use for background correction
+        bgshift_function : str
+            Background update strategy, either `pass` (static background) or `accurate`
+        strategy : str, optional
+            Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block`
+        '''
         if num_chunks > len(self.files) // 2:
             raise RuntimeError('Number of chunks exceeds more than half the number of files to process. Use less chunks.')
-        self.chunk_files(num_chunks)
+        self.chunk_files(num_chunks, strategy)
         self.build_initial_background_files(average_window=average_window)
         self.insert_bg_files_into_chunks(bgshift_function=bgshift_function)
 
-    def chunk_files(self, num_chunks: int):
+    def chunk_files(self, num_chunks: int, strategy: str = 'block'):
         '''Chunk the file list and create FilesToProcess.chunked_files
 
         Parameters
         ----------
         num_chunks : int
             number of chunks to produce (must be at least 1)
+        strategy : str, optional
+            Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block`
         '''
         if num_chunks < 1:
             raise RuntimeError('You must have at least one chunk')
         chunk_length = int(np.ceil(len(self.files) / num_chunks))
-        self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)]
+        if strategy == 'block':
+            logging.debug('Chunking file list with strategy: block')
+            self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)]
+        elif strategy == 'interleave':
+            logging.debug('Chunking file list with strategy: interleave')
+            self.chunked_files = [self.files[i::num_chunks] for i in range(0, num_chunks)]
+        else:
+            logging.debug(f'Invalid chunking strategy: {strategy}')
+            raise RuntimeError(f'Unknown strategy: {strategy}. Should be either block or interleave')
 
     def insert_bg_files_into_chunks(self, bgshift_function='pass'):
         average_window = len(self.background_files)

{pyopia-2.6.1 → pyopia-2.6.2}/pyopia/process.py

@@ -623,7 +623,7 @@ class CalculateImageStats():
 
         # Add image "global" statistics, separate from the particle stats above (stats)
         image_saturation = np.nan if stats.empty else stats['saturation'].values[0]
-        data['image_stats'].loc[data['timestamp'], 'filename'] = getattr(data, 'filename', '')
+        data['image_stats'].loc[data['timestamp'], 'filename'] = data.get('filename', '')
         data['image_stats'].loc[data['timestamp'], 'particle_count'] = int(stats.shape[0])
         data['image_stats'].loc[data['timestamp'], 'saturation'] = image_saturation
 

pyopia-2.6.1/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.6.1'