PyOPIA 2.9.3__tar.gz → 2.11.0__tar.gz

{pyopia-2.9.3 → pyopia-2.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyOPIA
-Version: 2.9.3
+Version: 2.11.0
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Project-URL: Repository, https://github.com/sintef/pyopia
 Project-URL: Documentation, https://pyopia.readthedocs.io
@@ -8,6 +8,7 @@ Author-email: Emlyn Davies <emlyn.davies@sintef.no>, "Alex Nimmo Smith@plymouth.
 License-File: LICENSE
 Keywords: Imaging,Measurement,Ocean,Particles,Size distribution
 Requires-Python: ~=3.12
+Requires-Dist: click<8.2.0
 Requires-Dist: cmocean<4,>=3.0.3
 Requires-Dist: dask>=2024.8.1
 Requires-Dist: flake8<7,>=6.1.0
@@ -21,14 +22,16 @@ Requires-Dist: matplotlib>=3.7
 Requires-Dist: myst-nb<0.18,>=0.17.2
 Requires-Dist: nbclient==0.7
 Requires-Dist: nbconvert<8,>=7.16.4
-Requires-Dist: numpy<2,>=1.24.0
+Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas[computation]<3,>=2.1.1
 Requires-Dist: poetry-version-plugin<0.3,>=0.2.0
+Requires-Dist: pydantic>=2.11.7
 Requires-Dist: pytest-error-for-skips<3,>=2.0.2
 Requires-Dist: pytest>=7.2.0
 Requires-Dist: readthedocs-sphinx-search<0.4,>=0.3.2
 Requires-Dist: scikit-image<0.25,>=0.24.0
 Requires-Dist: scipy<2,>=1.11.2
+Requires-Dist: seaborn>=0.13.2
 Requires-Dist: sphinx-copybutton<0.6,>=0.5.2
 Requires-Dist: sphinx-rtd-theme>=0.5.0
 Requires-Dist: sphinx-togglebutton<0.4,>=0.3.2
@@ -40,8 +43,8 @@ Requires-Dist: typer[all]<0.10,>=0.9.0
 Requires-Dist: urllib3<2.0
 Requires-Dist: xarray<2024,>=2023.12.0
 Provides-Extra: classification
-Requires-Dist: keras==3.5.0; extra == 'classification'
-Requires-Dist: tensorflow<3,>=2.16.2; extra == 'classification'
+Requires-Dist: keras==3.9.1; extra == 'classification'
+Requires-Dist: tensorflow>=2.19.0; extra == 'classification'
 Description-Content-Type: text/markdown
 
 PyOPIA
@@ -52,10 +55,15 @@ A Python Ocean Particle Image Analysis toolbox
 # Quick tryout of PyOPIA
 
 1) Install [uv](https://docs.astral.sh/uv/getting-started/installation)
-2) Run PyOPIA classification tests on database particles
+2) Initialize PyOPIA project with a small example image dataset and run processing
 ```bash
-uv run --python 3.12 --with git+https://github.com/SINTEF/pyopia --with tensorflow==2.16.2 --with keras==3.5.0 python -m pyopia.tests.test_classify
+uvx --python 3.12 --from pyopia[classification] pyopia --init-project pyopiatest --example-data
+cd pyopiatest
+uvx --python 3.12 --from pyopia[classification] pyopia process config.toml
 ```
+3) Inspect the processed particle statistics in the processed/ folder
+
+See the documentation for more information on how to install and use PyOPIA.
 
 # Documentation:
 

{pyopia-2.9.3 → pyopia-2.11.0}/README.md

@@ -6,10 +6,15 @@ A Python Ocean Particle Image Analysis toolbox
 # Quick tryout of PyOPIA
 
 1) Install [uv](https://docs.astral.sh/uv/getting-started/installation)
-2) Run PyOPIA classification tests on database particles
+2) Initialize PyOPIA project with a small example image dataset and run processing
 ```bash
-uv run --python 3.12 --with git+https://github.com/SINTEF/pyopia --with tensorflow==2.16.2 --with keras==3.5.0 python -m pyopia.tests.test_classify
+uvx --python 3.12 --from pyopia[classification] pyopia --init-project pyopiatest --example-data
+cd pyopiatest
+uvx --python 3.12 --from pyopia[classification] pyopia process config.toml
 ```
+3) Inspect the processed particle statistics in the processed/ folder
+
+See the documentation for more information on how to install and use PyOPIA.
 
 # Documentation:
 

pyopia-2.11.0/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.11.0"

pyopia-2.11.0/pyopia/auxillarydata.py

@@ -0,0 +1,106 @@
+import pandas as pd
+import logging
+import xarray as xr
+
+logger = logging.getLogger()
+
+AUXILLARY_DATA_FILE_TEMPLATE = """% COMMENT LINE: PLEASE UPDATE THIS FILE WITH PROJECT RELEVANT DATA. EACH COLUMN WILL BECOME A NETCDF VARIABLE.
+% COMMENT LINE: ONE LINE PER MEASUREMENT, TIME IS INTERPOLATED TO IMAGE DATA TIMES IN PYOPIA. FOLLOWING LINES ARE UNITS, DESCRIPTION AND VARIABLE NAME.
+,metres,degC
+Time of measurement,Depth at measurement location,Temperature at measurement location
+time,depth,temperature
+2022-06-08T18:40:00.00000,0.0,5.0
+2022-06-08T18:41:00.00000,5.0,6.0
+2022-06-08T18:42:00.00000,10.0,7.0
+2022-06-08T18:43:00.00000,20.0,8.0
+"""  # noqa: E501
+
+
+class AuxillaryData:
+    """
+    Handle auxillary data for PyOPIA particle statistics file.
+
+    Auxillary data variables may include (image) depth, longitude, latitude, etc.
+    This class parses a well-defined defined .csv input format, see example below.
+
+    Parameters
+    ----------
+    auxillary_data_path : str
+        Path to auxillary data files .csv creates by the enduser
+
+
+    Example of auxillary data file
+    ------------------------------
+    % COMMENT LINE:
+    % COMMENT LINE:
+    ,m,degC,psu
+    Time of measurement,Depth at measurement location,Temperature at measurement location
+    time,depth,temperature,salinity
+    2025-03-19T16:59:29.950729,0.0,5.0,34
+    2025-03-19T17:59:29.950729,0.0,5.0,34
+
+
+    Note
+    ----
+    Each column will become a netCDF variable
+    The two first lines are comments, and are ignored
+    Note that the third and fourth rows contain units and description for each variable
+    """
+
+    def __init__(self, auxillary_data_path=None):
+        self.auxillary_data_path = auxillary_data_path
+
+        # Create empty dataframe for cases where no file was specified, or an error occured reading it
+        self.auxillary_data = pd.DataFrame(index=pd.Index([], name="time")).to_xarray()
+        if auxillary_data_path is not None:
+            try:
+                self.auxillary_data = self.load_auxillary_data(auxillary_data_path)
+            except RuntimeError as e:
+                print(f"Failed to load auxillary data from file: {self.auxillary_data}")
+                logging.error(
+                    f"Failed to load auxillary data from file: {self.auxillary_data}"
+                )
+                logging.error(e)
+
+    def load_auxillary_data(self, auxillary_data_path):
+        """Load and format uxillary data from .csv file"""
+
+        # Load in the auxillary data file
+        auxillary_data = pd.read_csv(auxillary_data_path, skiprows=4)
+
+        # Load units and description rows
+        units = pd.read_csv(auxillary_data_path, skiprows=2, nrows=0).columns
+        long_names = pd.read_csv(auxillary_data_path, skiprows=3, nrows=0).columns
+
+        # Set time as the index and make sure its type is datetime64[ns]
+        auxillary_data["time"] = auxillary_data["time"].astype("datetime64[ns]")
+        auxillary_data = auxillary_data.set_index("time")
+
+        # Transform into xarray, add units
+        auxillary_data = auxillary_data.to_xarray()
+        for i, col in enumerate(auxillary_data.data_vars):  # Iterate over each column
+            auxillary_data[col].attrs["units"] = units[i + 1]
+            auxillary_data[col].attrs["long_name"] = long_names[i + 1]
+
+        logging.info(auxillary_data)
+
+        return auxillary_data
+
+    def add_auxillary_data_to_xstats(self, xstats):
+        """Add auxillary data to a PyOPIA xstats object"""
+        logging.info("Adding auxillary data to xstats and storing to new file")
+
+        # Add each auxillary data variable to xstats, interpolated to xstats times
+        for (
+            data_var
+        ) in self.auxillary_data.data_vars:  # Iterate over each data variable
+            xstats[data_var] = xr.DataArray(
+                data=self.auxillary_data[data_var]
+                .astype(float)
+                .interp(time=xstats["timestamp"]),
+                dims=xstats.dims,
+                coords=xstats.coords,
+                attrs=self.auxillary_data[data_var].attrs,
+            )
+
+        return xstats

{pyopia-2.9.3 → pyopia-2.11.0}/pyopia/cf_metadata.json

@@ -1,49 +1,49 @@
 {"major_axis_length": {
     "standard_name": "major_axis_length",
     "long_name": "The length of the major axis of the ellipse that has the same normalized second central moments as the region",
-    "units": "micrometer",
+    "units": "Pixels",
     "calculation_method": "Computed using skimage.measure.regionprops (axis_major_length)",
     "pyopia_process_level": 1},
 "minor_axis_length": {
 "standard_name": "minor_axis_length",
     "long_name": "The length of the minor axis of the ellipse that has the same normalized second central moments as the region",
-    "units": "micrometer",
+    "units": "Pixels",
     "calculation_method": "Computed using skimage.measure.regionprops (axis_minor_length)",
     "pyopia_process_level": 1},
 "equivalent_diameter": {
     "standard_name": "equivalent_circular_diameter",
     "long_name": "Diameter of a circle with the same area as the particle",
-    "units": "micrometer",
+    "units": "Pixels",
     "calculation_method": "Computed using skimage.measure.regionprops (equivalent_diameter)",
     "pyopia_process_level": 1},
 "minr": {
     "standard_name": "minimum_row_index",
     "long_name": "Minimum row index of the particle bounding box",
-    "units": "pixels",
+    "units": "Pixels",
     "calculation_method": "Extracted from skimage.measure.regionprops (bbox[0])",
     "pyopia_process_level": 1},
 "maxr": {
     "standard_name": "maximum_row_index",
     "long_name": "Maximum row index of the particle bounding box",
-    "units": "pixels",
+    "units": "Pixels",
     "calculation_method": "Extracted from skimage.measure.regionprops (bbox[2])",
     "pyopia_process_level": 1},
 "minc": {
     "standard_name": "minimum_column_index",
     "long_name": "Minimum column index of the particle bounding box",
-    "units": "pixels",
+    "units": "Pixels",
     "calculation_method": "Extracted from skimage.measure.regionprops (bbox[1])",
     "pyopia_process_level": 1},
 "maxc": {
     "standard_name": "maximum_column_index",
     "long_name": "Maximum column index of the particle bounding box",
-    "units": "pixels",
+    "units": "Pixels",
     "calculation_method": "Extracted from skimage.measure.regionprops (bbox[3])",
     "pyopia_process_level": 1},
 "saturation": {
     "standard_name": "image_saturation",
     "long_name": "Percentage saturation of the image",
-    "units": "percent",
+    "units": "Percent",
     "calculation_method": "Computed as the percentage of the image covered by particles relative to the maximum acceptable coverage",
     "pyopia_process_level": 1},
 "index": {
@@ -58,11 +58,6 @@
     "units": "",
     "calculation_method": "Generated during particle export",
     "pyopia_process_level": 1},
-"time": {
-    "standard_name": "time",
-    "long_name": "Time of particle observation",
-    "calculation_method": "Extracted from the timestamp of the observation",
-    "pyopia_process_level": 0},
 "timestamp": {
     "standard_name": "timestamp",
     "long_name": "Timestamp of particle observation",

{pyopia-2.9.3 → pyopia-2.11.0}/pyopia/classify.py

@@ -1,11 +1,14 @@
-'''
+"""
 Module containing tools for classifying particle ROIs
-'''
+"""
 
 import os
+import hashlib
 import numpy as np
 import pandas as pd
 import logging
+from skimage.exposure import rescale_intensity
+
 logger = logging.getLogger()
 
 # import tensorflow here. It must be imported on the processor where it will be used!
@@ -15,16 +18,16 @@ try:
     from tensorflow import keras
     import tensorflow as tf
 except ImportError:
-    info_str = 'ERROR: Could not import Keras. Classify will not work'
-    info_str += ' until you install tensorflow.\n'
-    info_str += 'Use: pip install pyopia[classification]\n'
-    info_str += ' or: pip install pyopia[classification-arm64]'
-    info_str += ' for tensorflow-macos (silicon chips)'
+    info_str = "ERROR: Could not import Keras. Classify will not work"
+    info_str += " until you install tensorflow.\n"
+    info_str += "Use: pip install pyopia[classification]\n"
+    info_str += " or: pip install pyopia[classification-arm64]"
+    info_str += " for tensorflow-macos (silicon chips)"
     raise ImportError(info_str)
 
 
-class Classify():
-    '''
+class Classify:
+    """
     A classifier class for PyOPIA workflow.
     This is intended as a parent class that can be used as a template for flexible classification methods
 
@@ -32,6 +35,10 @@ class Classify():
     ----------
     model_path : str
         path to particle-classifier e.g. '/testdata/model_name/particle_classifier.keras'
+    normalize_intensity : bool
+        Scale input image intensity to [0-1] range before classification
+    correct_whitebalance : bool
+        Perform whitebalance correction before classification
 
     Example
     -------
@@ -71,23 +78,36 @@ class Classify():
         import pyopia.exampledata
         model_path = exampledata.get_example_model()
 
-    '''
-    def __init__(self, model_path=None):
+    """
+
+    def __init__(
+        self,
+        model_path=None,
+        normalize_intensity=True,
+        correct_whitebalance=False,
+    ):
         self.model_path = model_path
         self.load_model()
 
         # Get config for image resizing from the model
-        _, self.img_height, self.img_width, _ = self.model.get_config()['layers'][0]['config']['batch_shape']
-        self.pad_to_aspect_ratio = getattr(self.model.layers[0], 'pad_to_aspect_ratio', False)
+        _, self.img_height, self.img_width, _ = self.model.get_config()["layers"][0][
+            "config"
+        ]["batch_shape"]
+        self.pad_to_aspect_ratio = getattr(
+            self.model.layers[0], "pad_to_aspect_ratio", False
+        )
 
         # Enable this to perform whitebalance correction in the preprocessing step
-        self.correct_whitebalance = False
+        self.correct_whitebalance = correct_whitebalance
+
+        # Enable this to rescale intensity in the preprocessing step
+        self.normalize_intensity = normalize_intensity
 
     def __call__(self):
         return self
 
     def load_model(self):
-        '''
+        """
         Load a trained Keras model into the Classify class.
 
         Parameters
@@ -96,34 +116,41 @@ class Classify():
             loaded Keras model
         class_names: list
             names for the model output classes
-        '''
+        """
         model_path = self.model_path
 
-        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+        os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
         keras.backend.clear_session()
 
         # Instantiate Keras model from file
         path, filename = os.path.split(model_path)
         self.model = keras.models.load_model(model_path)
 
+        # Create a hash of the model weights file
+        with open(model_path, "rb") as f:
+            digest = hashlib.file_digest(f, "sha256")
+        self.model_hash = digest.hexdigest()
+
         # Try to create model output class name list from last model layer name
         class_labels = None
         try:
-            class_labels = self.model.layers[-1].name.split('.')
+            class_labels = self.model.layers[-1].name.split(".")
         except:  # noqa E722
-            logger.info('Could not get class names from model layer name, reverting to old method with header file.')
+            logger.info(
+                "Could not get class names from model layer name, reverting to old method with header file."
+            )
 
         # If we could not create correct class names above, revert to old header file method
         expected_class_number = self.model.layers[-1].output.shape[1]
         if class_labels is None or len(class_labels) != expected_class_number:
-            header = pd.read_csv(os.path.join(path, 'header.tfl.txt'))
+            header = pd.read_csv(os.path.join(path, "header.tfl.txt"))
             class_labels = header.columns
 
         self.class_labels = class_labels
         logger.info(self.class_labels)
 
     def preprocessing(self, img_input):
-        '''
+        """
         Preprocess ROI ready for prediction. example here based on the pysilcam network setup
 
         Parameters
@@ -135,7 +162,7 @@ class Classify():
         -------
         img_preprocessed : float
             A particle ROI with range 0.-255., corrected and preprocessed, ready for prediction
-        '''
+        """
 
         whitebalanced = img_input.astype(np.float64)
 
@@ -143,10 +170,16 @@ class Classify():
         if self.correct_whitebalance:
             p = 99
             for c in range(3):
-                whitebalanced[:, :, c] += (p/100) - np.percentile(whitebalanced[:, :, c], p)
+                whitebalanced[:, :, c] += (p / 100) - np.percentile(
+                    whitebalanced[:, :, c], p
+                )
             whitebalanced[whitebalanced > 1] = 1
             whitebalanced[whitebalanced < 0] = 0
 
+        # Rescale intensity to 0-1 range
+        if self.normalize_intensity:
+            whitebalanced = rescale_intensity(whitebalanced)
+
         # convert back to 0-255 scaling (because of this layer in the network:
         # layers.Rescaling(1./255, input_shape=(img_height, img_width, 3)))
         # This is useful because it allows training to use tf.keras.utils.image_dataset_from_directory,
@@ -154,9 +187,12 @@ class Classify():
         img = keras.utils.img_to_array(whitebalanced * 255)
 
         # resize to match the dimentions expected by the network
-        img = tf.image.resize(img, [self.img_height, self.img_width],
-                              method=tf.image.ResizeMethod.BILINEAR,
-                              preserve_aspect_ratio=self.pad_to_aspect_ratio)
+        img = tf.image.resize(
+            img,
+            [self.img_height, self.img_width],
+            method=tf.image.ResizeMethod.BILINEAR,
+            preserve_aspect_ratio=self.pad_to_aspect_ratio,
+        )
 
         img_array = tf.keras.utils.img_to_array(img)
         img_preprocessed = tf.expand_dims(img_array, 0)  # Create a batch
@@ -164,7 +200,7 @@ class Classify():
 
     @tf.function
     def predict(self, img_preprocessed):
-        '''
+        """
         Use tensorflow model to classify particles. example here based on the pysilcam network setup.
 
         Parameters
@@ -177,14 +213,14 @@ class Classify():
         -------
         prediction : array
             The probability of the roi belonging to each class
-        '''
+        """
 
         prediction = self.model(img_preprocessed, training=False)
         prediction = tf.nn.softmax(prediction[0])
         return prediction
 
     def proc_predict(self, img_input):
-        '''
+        """
         Run pre-processing (:meth:`Classify.preprocessing`) and prediction (:meth:`Classify.predict`)
         using tensorflow model to classify particles. example here based on the pysilcam network setup.
 
@@ -197,7 +233,7 @@ class Classify():
         -------
         prediction : array
             The probability of the roi belonging to each class
-        '''
+        """
         img_preprocessed = self.preprocessing(img_input)
         prediction = self.predict(img_preprocessed)