PyOPIA 2.9.2__tar.gz → 2.10.0__tar.gz

{pyopia-2.9.2 → pyopia-2.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyOPIA
-Version: 2.9.2
+Version: 2.10.0
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Project-URL: Repository, https://github.com/sintef/pyopia
 Project-URL: Documentation, https://pyopia.readthedocs.io
@@ -8,6 +8,7 @@ Author-email: Emlyn Davies <emlyn.davies@sintef.no>, "Alex Nimmo Smith@plymouth.
 License-File: LICENSE
 Keywords: Imaging,Measurement,Ocean,Particles,Size distribution
 Requires-Python: ~=3.12
+Requires-Dist: click<8.2.0
 Requires-Dist: cmocean<4,>=3.0.3
 Requires-Dist: dask>=2024.8.1
 Requires-Dist: flake8<7,>=6.1.0
@@ -21,7 +22,7 @@ Requires-Dist: matplotlib>=3.7
 Requires-Dist: myst-nb<0.18,>=0.17.2
 Requires-Dist: nbclient==0.7
 Requires-Dist: nbconvert<8,>=7.16.4
-Requires-Dist: numpy<2,>=1.24.0
+Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas[computation]<3,>=2.1.1
 Requires-Dist: poetry-version-plugin<0.3,>=0.2.0
 Requires-Dist: pytest-error-for-skips<3,>=2.0.2
@@ -29,6 +30,7 @@ Requires-Dist: pytest>=7.2.0
 Requires-Dist: readthedocs-sphinx-search<0.4,>=0.3.2
 Requires-Dist: scikit-image<0.25,>=0.24.0
 Requires-Dist: scipy<2,>=1.11.2
+Requires-Dist: seaborn>=0.13.2
 Requires-Dist: sphinx-copybutton<0.6,>=0.5.2
 Requires-Dist: sphinx-rtd-theme>=0.5.0
 Requires-Dist: sphinx-togglebutton<0.4,>=0.3.2
@@ -40,8 +42,8 @@ Requires-Dist: typer[all]<0.10,>=0.9.0
 Requires-Dist: urllib3<2.0
 Requires-Dist: xarray<2024,>=2023.12.0
 Provides-Extra: classification
-Requires-Dist: keras==3.5.0; extra == 'classification'
-Requires-Dist: tensorflow<3,>=2.16.2; extra == 'classification'
+Requires-Dist: keras==3.9.1; extra == 'classification'
+Requires-Dist: tensorflow>=2.19.0; extra == 'classification'
 Description-Content-Type: text/markdown
 
 PyOPIA

pyopia-2.10.0/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.10.0"

pyopia-2.10.0/pyopia/cf_metadata.json

@@ -0,0 +1,70 @@
+{"major_axis_length": {
+    "standard_name": "major_axis_length",
+    "long_name": "The length of the major axis of the ellipse that has the same normalized second central moments as the region",
+    "units": "micrometer",
+    "calculation_method": "Computed using skimage.measure.regionprops (axis_major_length)",
+    "pyopia_process_level": 1},
+"minor_axis_length": {
+"standard_name": "minor_axis_length",
+    "long_name": "The length of the minor axis of the ellipse that has the same normalized second central moments as the region",
+    "units": "micrometer",
+    "calculation_method": "Computed using skimage.measure.regionprops (axis_minor_length)",
+    "pyopia_process_level": 1},
+"equivalent_diameter": {
+    "standard_name": "equivalent_circular_diameter",
+    "long_name": "Diameter of a circle with the same area as the particle",
+    "units": "micrometer",
+    "calculation_method": "Computed using skimage.measure.regionprops (equivalent_diameter)",
+    "pyopia_process_level": 1},
+"minr": {
+    "standard_name": "minimum_row_index",
+    "long_name": "Minimum row index of the particle bounding box",
+    "units": "pixels",
+    "calculation_method": "Extracted from skimage.measure.regionprops (bbox[0])",
+    "pyopia_process_level": 1},
+"maxr": {
+    "standard_name": "maximum_row_index",
+    "long_name": "Maximum row index of the particle bounding box",
+    "units": "pixels",
+    "calculation_method": "Extracted from skimage.measure.regionprops (bbox[2])",
+    "pyopia_process_level": 1},
+"minc": {
+    "standard_name": "minimum_column_index",
+    "long_name": "Minimum column index of the particle bounding box",
+    "units": "pixels",
+    "calculation_method": "Extracted from skimage.measure.regionprops (bbox[1])",
+    "pyopia_process_level": 1},
+"maxc": {
+    "standard_name": "maximum_column_index",
+    "long_name": "Maximum column index of the particle bounding box",
+    "units": "pixels",
+    "calculation_method": "Extracted from skimage.measure.regionprops (bbox[3])",
+    "pyopia_process_level": 1},
+"saturation": {
+    "standard_name": "image_saturation",
+    "long_name": "Percentage saturation of the image",
+    "units": "percent",
+    "calculation_method": "Computed as the percentage of the image covered by particles relative to the maximum acceptable coverage",
+    "pyopia_process_level": 1},
+"index": {
+    "standard_name": "index",
+    "long_name": "Index of the particle in the dataset",
+    "units": "",
+    "calculation_method": "Sequential numbering of particles in the dataset",
+    "pyopia_process_level": 1},
+"export_name": {
+    "standard_name": "export_name",
+    "long_name": "Name of the exported particle ROI file",
+    "units": "",
+    "calculation_method": "Generated during particle export",
+    "pyopia_process_level": 1},
+"time": {
+    "standard_name": "time",
+    "long_name": "Time of particle observation",
+    "calculation_method": "Extracted from the timestamp of the observation",
+    "pyopia_process_level": 0},
+"timestamp": {
+    "standard_name": "timestamp",
+    "long_name": "Timestamp of particle observation",
+    "calculation_method": "Recorded during particle observation",
+    "pyopia_process_level": 0}}

{pyopia-2.9.2 → pyopia-2.10.0}/pyopia/classify.py

@@ -1,11 +1,13 @@
-'''
+"""
 Module containing tools for classifying particle ROIs
-'''
+"""
 
 import os
 import numpy as np
 import pandas as pd
 import logging
+from skimage.exposure import rescale_intensity
+
 logger = logging.getLogger()
 
 # import tensorflow here. It must be imported on the processor where it will be used!
@@ -15,16 +17,16 @@ try:
     from tensorflow import keras
     import tensorflow as tf
 except ImportError:
-    info_str = 'ERROR: Could not import Keras. Classify will not work'
-    info_str += ' until you install tensorflow.\n'
-    info_str += 'Use: pip install pyopia[classification]\n'
-    info_str += ' or: pip install pyopia[classification-arm64]'
-    info_str += ' for tensorflow-macos (silicon chips)'
+    info_str = "ERROR: Could not import Keras. Classify will not work"
+    info_str += " until you install tensorflow.\n"
+    info_str += "Use: pip install pyopia[classification]\n"
+    info_str += " or: pip install pyopia[classification-arm64]"
+    info_str += " for tensorflow-macos (silicon chips)"
     raise ImportError(info_str)
 
 
-class Classify():
-    '''
+class Classify:
+    """
     A classifier class for PyOPIA workflow.
     This is intended as a parent class that can be used as a template for flexible classification methods
 
@@ -32,6 +34,10 @@ class Classify():
     ----------
     model_path : str
         path to particle-classifier e.g. '/testdata/model_name/particle_classifier.keras'
+    normalize_intensity : bool
+        Scale input image intensity to [0-1] range before classification
+    correct_whitebalance : bool
+        Perform whitebalance correction before classification
 
     Example
     -------
@@ -71,23 +77,36 @@ class Classify():
         import pyopia.exampledata
         model_path = exampledata.get_example_model()
 
-    '''
-    def __init__(self, model_path=None):
+    """
+
+    def __init__(
+        self,
+        model_path=None,
+        normalize_intensity=True,
+        correct_whitebalance=False,
+    ):
         self.model_path = model_path
         self.load_model()
 
         # Get config for image resizing from the model
-        _, self.img_height, self.img_width, _ = self.model.get_config()['layers'][0]['config']['batch_shape']
-        self.pad_to_aspect_ratio = getattr(self.model.layers[0], 'pad_to_aspect_ratio', False)
+        _, self.img_height, self.img_width, _ = self.model.get_config()["layers"][0][
+            "config"
+        ]["batch_shape"]
+        self.pad_to_aspect_ratio = getattr(
+            self.model.layers[0], "pad_to_aspect_ratio", False
+        )
 
         # Enable this to perform whitebalance correction in the preprocessing step
-        self.correct_whitebalance = False
+        self.correct_whitebalance = correct_whitebalance
+
+        # Enable this to rescale intensity in the preprocessing step
+        self.normalize_intensity = normalize_intensity
 
     def __call__(self):
         return self
 
     def load_model(self):
-        '''
+        """
         Load a trained Keras model into the Classify class.
 
         Parameters
@@ -96,10 +115,10 @@ class Classify():
             loaded Keras model
         class_names: list
             names for the model output classes
-        '''
+        """
         model_path = self.model_path
 
-        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+        os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
         keras.backend.clear_session()
 
         # Instantiate Keras model from file
@@ -109,21 +128,23 @@ class Classify():
         # Try to create model output class name list from last model layer name
         class_labels = None
         try:
-            class_labels = self.model.layers[-1].name.split('.')
+            class_labels = self.model.layers[-1].name.split(".")
         except:  # noqa E722
-            logger.info('Could not get class names from model layer name, reverting to old method with header file.')
+            logger.info(
+                "Could not get class names from model layer name, reverting to old method with header file."
+            )
 
         # If we could not create correct class names above, revert to old header file method
         expected_class_number = self.model.layers[-1].output.shape[1]
         if class_labels is None or len(class_labels) != expected_class_number:
-            header = pd.read_csv(os.path.join(path, 'header.tfl.txt'))
+            header = pd.read_csv(os.path.join(path, "header.tfl.txt"))
             class_labels = header.columns
 
         self.class_labels = class_labels
         logger.info(self.class_labels)
 
     def preprocessing(self, img_input):
-        '''
+        """
         Preprocess ROI ready for prediction. example here based on the pysilcam network setup
 
         Parameters
@@ -135,7 +156,7 @@ class Classify():
         -------
         img_preprocessed : float
             A particle ROI with range 0.-255., corrected and preprocessed, ready for prediction
-        '''
+        """
 
         whitebalanced = img_input.astype(np.float64)
 
@@ -143,10 +164,16 @@ class Classify():
         if self.correct_whitebalance:
             p = 99
             for c in range(3):
-                whitebalanced[:, :, c] += (p/100) - np.percentile(whitebalanced[:, :, c], p)
+                whitebalanced[:, :, c] += (p / 100) - np.percentile(
+                    whitebalanced[:, :, c], p
+                )
             whitebalanced[whitebalanced > 1] = 1
             whitebalanced[whitebalanced < 0] = 0
 
+        # Rescale intensity to 0-1 range
+        if self.normalize_intensity:
+            whitebalanced = rescale_intensity(whitebalanced)
+
         # convert back to 0-255 scaling (because of this layer in the network:
         # layers.Rescaling(1./255, input_shape=(img_height, img_width, 3)))
         # This is useful because it allows training to use tf.keras.utils.image_dataset_from_directory,
@@ -154,9 +181,12 @@ class Classify():
         img = keras.utils.img_to_array(whitebalanced * 255)
 
         # resize to match the dimentions expected by the network
-        img = tf.image.resize(img, [self.img_height, self.img_width],
-                              method=tf.image.ResizeMethod.BILINEAR,
-                              preserve_aspect_ratio=self.pad_to_aspect_ratio)
+        img = tf.image.resize(
+            img,
+            [self.img_height, self.img_width],
+            method=tf.image.ResizeMethod.BILINEAR,
+            preserve_aspect_ratio=self.pad_to_aspect_ratio,
+        )
 
         img_array = tf.keras.utils.img_to_array(img)
         img_preprocessed = tf.expand_dims(img_array, 0)  # Create a batch
@@ -164,7 +194,7 @@ class Classify():
 
     @tf.function
     def predict(self, img_preprocessed):
-        '''
+        """
         Use tensorflow model to classify particles. example here based on the pysilcam network setup.
 
         Parameters
@@ -177,14 +207,14 @@ class Classify():
         -------
         prediction : array
             The probability of the roi belonging to each class
-        '''
+        """
 
         prediction = self.model(img_preprocessed, training=False)
         prediction = tf.nn.softmax(prediction[0])
         return prediction
 
     def proc_predict(self, img_input):
-        '''
+        """
         Run pre-processing (:meth:`Classify.preprocessing`) and prediction (:meth:`Classify.predict`)
         using tensorflow model to classify particles. example here based on the pysilcam network setup.
 
@@ -197,7 +227,7 @@ class Classify():
         -------
         prediction : array
             The probability of the roi belonging to each class
-        '''
+        """
         img_preprocessed = self.preprocessing(img_input)
         prediction = self.predict(img_preprocessed)
 

pyopia-2.10.0/pyopia/exampledata.py

@@ -0,0 +1,150 @@
+import urllib.request
+import zipfile
+import os
+import gdown
+from pathlib import Path
+
+import logging
+
+logger = logging.getLogger()
+
+
+def get_classifier_database_from_pysilcam_blob(download_directory="./"):
+    """Downloads a specified filename from the pysilcam.blob into the working dir. if it doesn't already exist
+
+    only works for known filenames that are on this blob
+
+    Parameters
+    ----------
+    filename : string
+        known filename on the blob
+
+    """
+    if os.path.exists(os.path.join(download_directory)):
+        logger.info(download_directory, "already exists. Returning nothing")
+        return download_directory
+    os.makedirs(download_directory, exist_ok=False)
+    url = "https://pysilcam.blob.core.windows.net/test-data/silcam_database.zip"
+    logger.info("Downloading....")
+    urllib.request.urlretrieve(url, download_directory + "/silcam_database.zip")
+    logger.info("Unzipping....")
+    with zipfile.ZipFile(
+        os.path.join(download_directory, "silcam_database.zip"), "r"
+    ) as zipit:
+        zipit.extractall(os.path.join(download_directory, "../"))
+    logger.info("Removing zip file")
+    os.remove(os.path.join(download_directory, "silcam_database.zip"))
+    logger.info("Done.")
+    return download_directory
+
+
+def get_file_from_pysilcam_blob(filename, download_directory="./"):
+    """Downloads a specified filename from the pysilcam.blob into the working dir. if it doesn't already exist
+
+    only works for known filenames that are on this blob
+
+    Parameters
+    ----------
+    filename : string
+        known filename on the blob
+
+    """
+    if os.path.exists(os.path.join(download_directory, filename)):
+        return filename
+    url = "https://pysilcam.blob.core.windows.net/test-data/" + filename
+    urllib.request.urlretrieve(url, os.path.join(download_directory, filename))
+    return download_directory
+
+
+def get_example_silc_image(download_directory="./"):
+    """calls `get_file_from_pysilcam_blob` for a silcam iamge
+
+    Returns
+    -------
+    string
+        filename
+    """
+    filename = "D20181101T142731.838206.silc"
+    if os.path.isfile(filename):
+        logger.info("Example image already exists. Skipping download.")
+        return filename
+    get_file_from_pysilcam_blob(filename, download_directory)
+    return filename
+
+
+def get_example_model(download_directory="./"):
+    """Download PyOPIA default CNN model classifier
+
+    Download from the pysilcam blob storage into the working dir.
+    If the file exists, skip the download.
+
+    Returns
+    -------
+    string
+        model_filename
+    """
+    model_filename = "pyopia-default-classifier-20250409.keras"
+    model_path = Path(download_directory, model_filename)
+    model_url = (
+        f"https://pysilcam.blob.core.windows.net/test-data/{str(model_filename)}"
+    )
+    if not model_path.exists():
+        logger.info("Downloading example model...")
+        urllib.request.urlretrieve(model_url, model_path)
+    return str(model_path)
+
+
+def get_example_hologram_and_background(download_directory="./"):
+    """calls `get_file_from_pysilcam_blob` for a raw hologram, and its associated background image.
+
+    Returns
+    -------
+    string
+        holo_filename
+
+    string
+        holo_background_filename
+    """
+    holo_filename = "001-2082.pgm"
+    holo_background_filename = "imbg-" + holo_filename
+    get_file_from_pysilcam_blob(holo_filename, download_directory)
+    get_file_from_pysilcam_blob(holo_background_filename, download_directory)
+
+    holo_filename = os.path.join(download_directory, holo_filename)
+    holo_background_filename = os.path.join(
+        download_directory, holo_background_filename
+    )
+
+    return holo_filename, holo_background_filename
+
+
+def get_folder_from_holo_repository(foldername="holo_test_data_01", existsok=False):
+    """Downloads a specified folder from the holo testing repository into the working dir. if it doesn't already exist
+
+    only works for known folders that are on the GoogleDrive repository
+    by default will download a known-good folder. Additional elif statements can be added to implement additional folders.
+
+    Parameters
+    ----------
+    foldername : string
+        known filename on the blob
+    existsok : (bool, optional)
+        if True, then don't download if the specified folder already exists, defaults to False
+
+    """
+    if foldername == "holo_test_data_01":
+        url = "https://drive.google.com/drive/folders/1yNatOaKdWwYQp-5WVEDItoibr-k0lGsP?usp=share_link"
+
+    elif foldername == "holo_test_data_02":
+        url = "https://drive.google.com/drive/folders/1E5iNSyfeKcVMLVe4PNEwF2Q2mo3WVjF5?usp=share_link"
+
+    else:
+        foldername == "holo_test_data_01"
+        url = "https://drive.google.com/drive/folders/1yNatOaKdWwYQp-5WVEDItoibr-k0lGsP?usp=share_link"
+
+    if os.path.exists(foldername) and existsok:
+        logger.info(foldername + " already exists. Skipping download.")
+        return foldername
+
+    gdown.download_folder(url, quiet=True, use_cookies=False)
+    return foldername