PyOPIA 2.0.4__tar.gz → 2.1.0__tar.gz

{pyopia-2.0.4 → pyopia-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyOPIA
-Version: 2.0.4
+Version: 2.1.0
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Home-page: https://github.com/sintef/pyopia
 Keywords: Ocean,Particles,Imaging,Measurement,Size distribution

pyopia-2.1.0/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.1.0'

{pyopia-2.0.4 → pyopia-2.1.0}/pyopia/cli.py

@@ -147,6 +147,23 @@ def process(config_filename: str):
             logger.debug(''.join(traceback.format_tb(e.__traceback__)))
 
 
+@app.command()
+def merge_mfdata(path_to_data: str, prefix='*'):
+    '''Combine a multi-file directory of STATS.nc files into a single '-STATS.nc' file
+    that can then be loaded with {func}`pyopia.io.load_stats`
+
+    Parameters
+    ----------
+    path_to_data : str
+        Folder name containing nc files with pattern '*Image-D*-STATS.nc'
+
+    prefix : str
+        Prefix to multi-file dataset (for replacing the wildcard in '*Image-D*-STATS.nc').
+        Defaults to '*'
+    '''
+    pyopia.io.merge_and_save_mfdataset(path_to_data, prefix=prefix)
+
+
 def setup_logging(pipeline_config):
     '''Configure logging
 

{pyopia-2.0.4 → pyopia-2.1.0}/pyopia/instrument/silcam.py

@@ -36,7 +36,7 @@ def load_image(filename):
     array
         raw image
     '''
-    img = np.load(filename, allow_pickle=False).astype(np.uint8)
+    img = np.load(filename, allow_pickle=False).astype(np.float64)
     return img
 
 

{pyopia-2.0.4 → pyopia-2.1.0}/pyopia/io.py

@@ -10,6 +10,7 @@ import toml
 import xarray
 import os
 from glob import glob
+import xarray as xr
 
 from pyopia import __version__ as pyopia_version
 
@@ -32,15 +33,17 @@ def write_stats(stats,
     Appends if file already exists.
 
     Args:
-        datafilename (str):     filame prefix for -STATS.h5 file that may or may not include a path
-        stats_all (DataFrame):  stats dataframe returned from processImage()
-        export_name_len (int):  max number of chars allowed for col 'export name'
-        append (bool):          Append all processed data into one nc file.
-                                Defaults to True.
-                                If False, then one nc file will be generated per raw image,
-                                which can be loaded using :func:`pyopia.io.combine_stats_netcdf_files`
-                                This is useful for larger datasets, where appending causes substantial slowdown
-                                as the dataset gets larger.
+        datafilename (str)                      :   Filame prefix for -STATS.h5 file that may or may not include a path
+        stats (DataFrame or xr.Dataset)         :   STATS dataframe
+        export_name_len (int)                   :   Max number of chars allowed for col 'export name'
+        append (bool)                           :   Append all processed data into one nc file.
+                                                    Defaults to True.
+                                                    If False, then one nc file will be generated per raw image,
+                                                    which can be loaded using :func:`pyopia.io.combine_stats_netcdf_files`
+                                                    This is useful for larger datasets,
+                                                    where appending causes substantial slowdown
+                                                    as the dataset gets larger.
+        image_stats (xr.Dataset)                :   image_stats data
     '''
 
     if len(stats) == 0:  # to avoid issue with wrong time datatypes in xarray
@@ -64,7 +67,12 @@ def write_stats(stats,
             meta.attrs['PyOpia version'] = pyopia_version
             meta.attrs['Pipeline steps'] = settings
     elif dataformat == 'nc':
-        xstats = make_xstats(stats, settings)
+
+        if isinstance(stats, xr.Dataset):
+            xstats = stats
+        else:
+            xstats = make_xstats(stats, settings)
+
         if append and os.path.isfile(datafilename + '-STATS.nc'):
             existing_stats = load_stats(datafilename + '-STATS.nc')
             xstats = xarray.concat([existing_stats, xstats], 'index')
@@ -147,7 +155,7 @@ def load_stats(datafilename):
     return stats
 
 
-def combine_stats_netcdf_files(path_to_data):
+def combine_stats_netcdf_files(path_to_data, prefix='*'):
     '''Combine a multi-file directory of STATS.nc files into a 'stats' xarray dataset created by :func:`pyopia.io.write_stats`
     when using 'append = false'
 
@@ -156,14 +164,20 @@ def combine_stats_netcdf_files(path_to_data):
     path_to_data : str
         Folder name containing nc files with pattern '*Image-D*-STATS.nc'
 
+    prefix : str
+        Prefix to multi-file dataset (for replacing the wildcard in '*Image-D*-STATS.nc').
+        Defaults to '*'
+
     Returns
     -------
-    DataFrame
-        STATS xarray dataset
+    tuple
+        xstats STATS xarray dataset, image_stats dataset
     '''
 
-    sorted_filelist = sorted(glob(os.path.join(path_to_data, '*Image-D*-STATS.nc')))
-    with xarray.open_mfdataset(sorted_filelist, combine='nested', concat_dim='index') as ds:
+    sorted_filelist = sorted(glob(os.path.join(path_to_data, prefix + 'Image-D*-STATS.nc')))
+    with xarray.open_mfdataset(sorted_filelist, combine='nested', concat_dim='index',
+                               decode_cf=False, parallel=False,
+                               coords='minimal', compat='override') as ds:
         xstats = ds.load()
 
     # Check if we have image statistics in the last file, if so, load it.
@@ -180,6 +194,53 @@ def combine_stats_netcdf_files(path_to_data):
     return xstats, image_stats
 
 
+def merge_and_save_mfdataset(path_to_data, prefix='*'):
+    '''Combine a multi-file directory of STATS.nc files into a single '-STATS.nc' file
+    that can then be loaded with {func}`pyopia.io.load_stats`
+
+    Parameters
+    ----------
+    path_to_data : str
+        Folder name containing nc files with pattern '*Image-D*-STATS.nc'
+
+    prefix : str
+        Prefix to multi-file dataset (for replacing the wildcard in '*Image-D*-STATS.nc').
+        Defaults to '*'
+    '''
+
+    logging.info(f'combine stats netcdf files from {path_to_data}')
+    xstats, image_stats = combine_stats_netcdf_files(path_to_data, prefix=prefix)
+
+    settings = steps_from_xstats(xstats)
+
+    prefix_out = os.path.basename(settings['steps']['output']['output_datafile'])
+    output_name = os.path.join(path_to_data, prefix_out)
+
+    logging.info(f'writing {output_name}')
+    write_stats(xstats,
+                output_name,
+                settings,
+                image_stats=image_stats)
+    logging.info(f'writing {output_name} done.')
+
+
+def steps_from_xstats(xstats):
+    '''Get the steps attribute from xarray version of the particle stats into a dictionary
+
+    Parameters
+    ----------
+    xstats : xarray.DataSet
+        xarray version of the particle stats dataframe, containing metadata
+
+    Returns
+    -------
+    dict
+        TOML-formatted dictionary of pipeline steps
+    '''
+    steps = toml.loads(xstats.__getattr__('steps'))
+    return steps
+
+
 def load_stats_as_dataframe(stats_file):
     '''A loading function for stats files that forces stats into a pandas DataFrame
 

{pyopia-2.0.4 → pyopia-2.1.0}/pyopia/pipeline.py

@@ -7,8 +7,8 @@ from typing import TypedDict
 import datetime
 import pandas as pd
 from operator import methodcaller
-import toml
 import sys
+from pyopia.io import steps_from_xstats as steps_from_xstats # noqa: E(F401)
 import logging
 
 logger = logging.getLogger()
@@ -273,23 +273,6 @@ def steps_to_string(steps):
     return steps_str
 
 
-def steps_from_xstats(xstats):
-    '''Get the steps attribute from xarray version of the particle stats into a dictionary
-
-    Parameters
-    ----------
-    xstats : xarray.DataSet
-        xarray version of the particle stats dataframe, containing metadata
-
-    Returns
-    -------
-    dict
-        TOML-formatted dictionary of pipeline steps
-    '''
-    steps = toml.loads(xstats.__getattr__('steps'))
-    return steps
-
-
 def build_repr(toml_steps, step_name):
     '''Build a callable object from settings, which can be used to construct the pipeline steps dict
 

pyopia-2.0.4/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.0.4'