PyOPIA 2.9.3__tar.gz → 2.10.0__tar.gz

{pyopia-2.9.3 → pyopia-2.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyOPIA
-Version: 2.9.3
+Version: 2.10.0
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Project-URL: Repository, https://github.com/sintef/pyopia
 Project-URL: Documentation, https://pyopia.readthedocs.io
@@ -8,6 +8,7 @@ Author-email: Emlyn Davies <emlyn.davies@sintef.no>, "Alex Nimmo Smith@plymouth.
 License-File: LICENSE
 Keywords: Imaging,Measurement,Ocean,Particles,Size distribution
 Requires-Python: ~=3.12
+Requires-Dist: click<8.2.0
 Requires-Dist: cmocean<4,>=3.0.3
 Requires-Dist: dask>=2024.8.1
 Requires-Dist: flake8<7,>=6.1.0
@@ -21,7 +22,7 @@ Requires-Dist: matplotlib>=3.7
 Requires-Dist: myst-nb<0.18,>=0.17.2
 Requires-Dist: nbclient==0.7
 Requires-Dist: nbconvert<8,>=7.16.4
-Requires-Dist: numpy<2,>=1.24.0
+Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas[computation]<3,>=2.1.1
 Requires-Dist: poetry-version-plugin<0.3,>=0.2.0
 Requires-Dist: pytest-error-for-skips<3,>=2.0.2
@@ -29,6 +30,7 @@ Requires-Dist: pytest>=7.2.0
 Requires-Dist: readthedocs-sphinx-search<0.4,>=0.3.2
 Requires-Dist: scikit-image<0.25,>=0.24.0
 Requires-Dist: scipy<2,>=1.11.2
+Requires-Dist: seaborn>=0.13.2
 Requires-Dist: sphinx-copybutton<0.6,>=0.5.2
 Requires-Dist: sphinx-rtd-theme>=0.5.0
 Requires-Dist: sphinx-togglebutton<0.4,>=0.3.2
@@ -40,8 +42,8 @@ Requires-Dist: typer[all]<0.10,>=0.9.0
 Requires-Dist: urllib3<2.0
 Requires-Dist: xarray<2024,>=2023.12.0
 Provides-Extra: classification
-Requires-Dist: keras==3.5.0; extra == 'classification'
-Requires-Dist: tensorflow<3,>=2.16.2; extra == 'classification'
+Requires-Dist: keras==3.9.1; extra == 'classification'
+Requires-Dist: tensorflow>=2.19.0; extra == 'classification'
 Description-Content-Type: text/markdown
 
 PyOPIA

pyopia-2.10.0/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.10.0"

{pyopia-2.9.3 → pyopia-2.10.0}/pyopia/classify.py

@@ -1,11 +1,13 @@
-'''
+"""
 Module containing tools for classifying particle ROIs
-'''
+"""
 
 import os
 import numpy as np
 import pandas as pd
 import logging
+from skimage.exposure import rescale_intensity
+
 logger = logging.getLogger()
 
 # import tensorflow here. It must be imported on the processor where it will be used!
@@ -15,16 +17,16 @@ try:
     from tensorflow import keras
     import tensorflow as tf
 except ImportError:
-    info_str = 'ERROR: Could not import Keras. Classify will not work'
-    info_str += ' until you install tensorflow.\n'
-    info_str += 'Use: pip install pyopia[classification]\n'
-    info_str += ' or: pip install pyopia[classification-arm64]'
-    info_str += ' for tensorflow-macos (silicon chips)'
+    info_str = "ERROR: Could not import Keras. Classify will not work"
+    info_str += " until you install tensorflow.\n"
+    info_str += "Use: pip install pyopia[classification]\n"
+    info_str += " or: pip install pyopia[classification-arm64]"
+    info_str += " for tensorflow-macos (silicon chips)"
     raise ImportError(info_str)
 
 
-class Classify():
-    '''
+class Classify:
+    """
     A classifier class for PyOPIA workflow.
     This is intended as a parent class that can be used as a template for flexible classification methods
 
@@ -32,6 +34,10 @@ class Classify():
     ----------
     model_path : str
         path to particle-classifier e.g. '/testdata/model_name/particle_classifier.keras'
+    normalize_intensity : bool
+        Scale input image intensity to [0-1] range before classification
+    correct_whitebalance : bool
+        Perform whitebalance correction before classification
 
     Example
     -------
@@ -71,23 +77,36 @@ class Classify():
         import pyopia.exampledata
         model_path = exampledata.get_example_model()
 
-    '''
-    def __init__(self, model_path=None):
+    """
+
+    def __init__(
+        self,
+        model_path=None,
+        normalize_intensity=True,
+        correct_whitebalance=False,
+    ):
         self.model_path = model_path
         self.load_model()
 
         # Get config for image resizing from the model
-        _, self.img_height, self.img_width, _ = self.model.get_config()['layers'][0]['config']['batch_shape']
-        self.pad_to_aspect_ratio = getattr(self.model.layers[0], 'pad_to_aspect_ratio', False)
+        _, self.img_height, self.img_width, _ = self.model.get_config()["layers"][0][
+            "config"
+        ]["batch_shape"]
+        self.pad_to_aspect_ratio = getattr(
+            self.model.layers[0], "pad_to_aspect_ratio", False
+        )
 
         # Enable this to perform whitebalance correction in the preprocessing step
-        self.correct_whitebalance = False
+        self.correct_whitebalance = correct_whitebalance
+
+        # Enable this to rescale intensity in the preprocessing step
+        self.normalize_intensity = normalize_intensity
 
     def __call__(self):
         return self
 
     def load_model(self):
-        '''
+        """
         Load a trained Keras model into the Classify class.
 
         Parameters
@@ -96,10 +115,10 @@ class Classify():
             loaded Keras model
         class_names: list
             names for the model output classes
-        '''
+        """
         model_path = self.model_path
 
-        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+        os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
         keras.backend.clear_session()
 
         # Instantiate Keras model from file
@@ -109,21 +128,23 @@ class Classify():
         # Try to create model output class name list from last model layer name
         class_labels = None
         try:
-            class_labels = self.model.layers[-1].name.split('.')
+            class_labels = self.model.layers[-1].name.split(".")
         except:  # noqa E722
-            logger.info('Could not get class names from model layer name, reverting to old method with header file.')
+            logger.info(
+                "Could not get class names from model layer name, reverting to old method with header file."
+            )
 
         # If we could not create correct class names above, revert to old header file method
         expected_class_number = self.model.layers[-1].output.shape[1]
         if class_labels is None or len(class_labels) != expected_class_number:
-            header = pd.read_csv(os.path.join(path, 'header.tfl.txt'))
+            header = pd.read_csv(os.path.join(path, "header.tfl.txt"))
             class_labels = header.columns
 
         self.class_labels = class_labels
         logger.info(self.class_labels)
 
     def preprocessing(self, img_input):
-        '''
+        """
         Preprocess ROI ready for prediction. example here based on the pysilcam network setup
 
         Parameters
@@ -135,7 +156,7 @@ class Classify():
         -------
         img_preprocessed : float
             A particle ROI with range 0.-255., corrected and preprocessed, ready for prediction
-        '''
+        """
 
         whitebalanced = img_input.astype(np.float64)
 
@@ -143,10 +164,16 @@ class Classify():
         if self.correct_whitebalance:
             p = 99
             for c in range(3):
-                whitebalanced[:, :, c] += (p/100) - np.percentile(whitebalanced[:, :, c], p)
+                whitebalanced[:, :, c] += (p / 100) - np.percentile(
+                    whitebalanced[:, :, c], p
+                )
             whitebalanced[whitebalanced > 1] = 1
             whitebalanced[whitebalanced < 0] = 0
 
+        # Rescale intensity to 0-1 range
+        if self.normalize_intensity:
+            whitebalanced = rescale_intensity(whitebalanced)
+
         # convert back to 0-255 scaling (because of this layer in the network:
         # layers.Rescaling(1./255, input_shape=(img_height, img_width, 3)))
         # This is useful because it allows training to use tf.keras.utils.image_dataset_from_directory,
@@ -154,9 +181,12 @@ class Classify():
         img = keras.utils.img_to_array(whitebalanced * 255)
 
         # resize to match the dimentions expected by the network
-        img = tf.image.resize(img, [self.img_height, self.img_width],
-                              method=tf.image.ResizeMethod.BILINEAR,
-                              preserve_aspect_ratio=self.pad_to_aspect_ratio)
+        img = tf.image.resize(
+            img,
+            [self.img_height, self.img_width],
+            method=tf.image.ResizeMethod.BILINEAR,
+            preserve_aspect_ratio=self.pad_to_aspect_ratio,
+        )
 
         img_array = tf.keras.utils.img_to_array(img)
         img_preprocessed = tf.expand_dims(img_array, 0)  # Create a batch
@@ -164,7 +194,7 @@ class Classify():
 
     @tf.function
     def predict(self, img_preprocessed):
-        '''
+        """
         Use tensorflow model to classify particles. example here based on the pysilcam network setup.
 
         Parameters
@@ -177,14 +207,14 @@ class Classify():
         -------
         prediction : array
             The probability of the roi belonging to each class
-        '''
+        """
 
         prediction = self.model(img_preprocessed, training=False)
         prediction = tf.nn.softmax(prediction[0])
         return prediction
 
     def proc_predict(self, img_input):
-        '''
+        """
         Run pre-processing (:meth:`Classify.preprocessing`) and prediction (:meth:`Classify.predict`)
         using tensorflow model to classify particles. example here based on the pysilcam network setup.
 
@@ -197,7 +227,7 @@ class Classify():
         -------
         prediction : array
             The probability of the roi belonging to each class
-        '''
+        """
         img_preprocessed = self.preprocessing(img_input)
         prediction = self.predict(img_preprocessed)
 

pyopia-2.10.0/pyopia/exampledata.py

@@ -0,0 +1,150 @@
+import urllib.request
+import zipfile
+import os
+import gdown
+from pathlib import Path
+
+import logging
+
+logger = logging.getLogger()
+
+
+def get_classifier_database_from_pysilcam_blob(download_directory="./"):
+    """Downloads a specified filename from the pysilcam.blob into the working dir. if it doesn't already exist
+
+    only works for known filenames that are on this blob
+
+    Parameters
+    ----------
+    filename : string
+        known filename on the blob
+
+    """
+    if os.path.exists(os.path.join(download_directory)):
+        logger.info(download_directory, "already exists. Returning nothing")
+        return download_directory
+    os.makedirs(download_directory, exist_ok=False)
+    url = "https://pysilcam.blob.core.windows.net/test-data/silcam_database.zip"
+    logger.info("Downloading....")
+    urllib.request.urlretrieve(url, download_directory + "/silcam_database.zip")
+    logger.info("Unzipping....")
+    with zipfile.ZipFile(
+        os.path.join(download_directory, "silcam_database.zip"), "r"
+    ) as zipit:
+        zipit.extractall(os.path.join(download_directory, "../"))
+    logger.info("Removing zip file")
+    os.remove(os.path.join(download_directory, "silcam_database.zip"))
+    logger.info("Done.")
+    return download_directory
+
+
+def get_file_from_pysilcam_blob(filename, download_directory="./"):
+    """Downloads a specified filename from the pysilcam.blob into the working dir. if it doesn't already exist
+
+    only works for known filenames that are on this blob
+
+    Parameters
+    ----------
+    filename : string
+        known filename on the blob
+
+    """
+    if os.path.exists(os.path.join(download_directory, filename)):
+        return filename
+    url = "https://pysilcam.blob.core.windows.net/test-data/" + filename
+    urllib.request.urlretrieve(url, os.path.join(download_directory, filename))
+    return download_directory
+
+
+def get_example_silc_image(download_directory="./"):
+    """calls `get_file_from_pysilcam_blob` for a silcam iamge
+
+    Returns
+    -------
+    string
+        filename
+    """
+    filename = "D20181101T142731.838206.silc"
+    if os.path.isfile(filename):
+        logger.info("Example image already exists. Skipping download.")
+        return filename
+    get_file_from_pysilcam_blob(filename, download_directory)
+    return filename
+
+
+def get_example_model(download_directory="./"):
+    """Download PyOPIA default CNN model classifier
+
+    Download from the pysilcam blob storage into the working dir.
+    If the file exists, skip the download.
+
+    Returns
+    -------
+    string
+        model_filename
+    """
+    model_filename = "pyopia-default-classifier-20250409.keras"
+    model_path = Path(download_directory, model_filename)
+    model_url = (
+        f"https://pysilcam.blob.core.windows.net/test-data/{str(model_filename)}"
+    )
+    if not model_path.exists():
+        logger.info("Downloading example model...")
+        urllib.request.urlretrieve(model_url, model_path)
+    return str(model_path)
+
+
+def get_example_hologram_and_background(download_directory="./"):
+    """calls `get_file_from_pysilcam_blob` for a raw hologram, and its associated background image.
+
+    Returns
+    -------
+    string
+        holo_filename
+
+    string
+        holo_background_filename
+    """
+    holo_filename = "001-2082.pgm"
+    holo_background_filename = "imbg-" + holo_filename
+    get_file_from_pysilcam_blob(holo_filename, download_directory)
+    get_file_from_pysilcam_blob(holo_background_filename, download_directory)
+
+    holo_filename = os.path.join(download_directory, holo_filename)
+    holo_background_filename = os.path.join(
+        download_directory, holo_background_filename
+    )
+
+    return holo_filename, holo_background_filename
+
+
+def get_folder_from_holo_repository(foldername="holo_test_data_01", existsok=False):
+    """Downloads a specified folder from the holo testing repository into the working dir. if it doesn't already exist
+
+    only works for known folders that are on the GoogleDrive repository
+    by default will download a known-good folder. Additional elif statements can be added to implement additional folders.
+
+    Parameters
+    ----------
+    foldername : string
+        known filename on the blob
+    existsok : (bool, optional)
+        if True, then don't download if the specified folder already exists, defaults to False
+
+    """
+    if foldername == "holo_test_data_01":
+        url = "https://drive.google.com/drive/folders/1yNatOaKdWwYQp-5WVEDItoibr-k0lGsP?usp=share_link"
+
+    elif foldername == "holo_test_data_02":
+        url = "https://drive.google.com/drive/folders/1E5iNSyfeKcVMLVe4PNEwF2Q2mo3WVjF5?usp=share_link"
+
+    else:
+        foldername == "holo_test_data_01"
+        url = "https://drive.google.com/drive/folders/1yNatOaKdWwYQp-5WVEDItoibr-k0lGsP?usp=share_link"
+
+    if os.path.exists(foldername) and existsok:
+        logger.info(foldername + " already exists. Skipping download.")
+        return foldername
+
+    gdown.download_folder(url, quiet=True, use_cookies=False)
+    return foldername

pyopia-2.10.0/pyopia/plotting.py

@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+"""
+Particle plotting functionality for standardised figures
+e.g. image presentation, size distributions, montages etc.
+"""
+
+import matplotlib.pyplot as plt
+import numpy as np
+import seaborn as sns
+import pandas as pd
+import pyopia.statistics
+
+
+def show_image(image, pixel_size):
+    """Plots a scaled figure (in mm) of an image
+
+    Parameters
+    ----------
+    image : float
+        Image (usually a corrected image, such as im_corrected)
+    pixel_size : float
+        the pixel size (um) of the imaging system used
+    """
+    r, c = np.shape(image[:, :, 0])
+
+    plt.imshow(
+        image,
+        extent=[0, c * pixel_size / 1000, 0, r * pixel_size / 1000],
+        interpolation="nearest",
+    )
+    plt.xlabel("mm")
+    plt.ylabel("mm")
+
+    return
+
+
+def montage_plot(montage, pixel_size):
+    """
+    Plots a SilCam particle montage with a 1mm scale reference
+
+    Parameters
+    ----------
+    montage : uint8
+        a montage created with scpp.make_montage
+    pixel_size : float
+        the pixel size (um) of the imaging system used
+    """
+    msize = np.shape(montage)[0]
+    ex = pixel_size * np.float64(msize) / 1000.0
+
+    ax = plt.gca()
+    ax.imshow(montage, extent=[0, ex, 0, ex], cmap="grey")
+    ax.set_xticks([1, 2], [])
+    ax.set_xticklabels(["    1mm", ""])
+    ax.set_yticks([], [])
+    ax.xaxis.set_ticks_position("bottom")
+
+
+def classify_rois(roilist, classifier):
+    """Classify list of single-object images
+
+    If true_class is specified, mark ROIs not matching this class in the figure.
+
+    Parameters
+    ----------
+    roilist: list
+        List of ROI images to classify
+
+    classifier: pyopia.classify.Classify
+        Used to classify ROIs
+
+    Returns
+    -------
+    df: pd.DataFrame
+        Class probabilities for each item in roifiles
+    """
+
+    # Get class labels from classifier
+    class_labels = [f"probability_{cl}" for cl in classifier.class_labels]
+
+    # Classify all ROIs
+    classify_data = []
+    for img in roilist:
+        prediction = classifier.proc_predict(img).numpy()
+        classify_data.append(prediction)
+    df = pd.DataFrame(columns=class_labels, data=classify_data)
+
+    return df
+
+
+def plot_classified_rois(roilist, df_class_labels, true_class=None):
+    """Plot classified single-object images and show them in a figure grid with classification info
+
+    If true_class is specified, mark ROIs not matching this class in the figure.
+
+    Parameters
+    ----------
+    roilist: list
+        List of ROI image to classify
+
+    df_class_labels: pd.DataFrame
+        Class label for each image in roilist in a column named "best guess"
+
+    true_class: str
+        True class of listed ROIs
+
+
+    Returns
+    -------
+    fig: matplotlib figure
+    ax: matplotlib axes
+    """
+    if "best guess" not in df_class_labels:
+        raise RuntimeError(
+            "df_class_clabels must contain column 'best guess'. "
+            "See e.g. pyopia.statistics.add_best_guesses_to_stats"
+        )
+
+    # Get class labels and class index for each ROI
+    label_maxprob_list = df_class_labels["best guess"].values
+    class_maxprob_list = pd.Categorical(
+        df_class_labels["best guess"].values,
+        categories=[
+            cl
+            for cl in df_class_labels.columns
+            if cl not in ["best guess", "best guess value"]
+        ],
+        ordered=True,
+    ).codes
+
+    # Set up figure with 15 axes columns
+    N = len(roilist)
+    ncols = img_per_col = min(N, 15)
+    nrows = 1
+    if N > img_per_col:
+        ncols = img_per_col
+        nrows = N // ncols + int((N % img_per_col) > 0)
+    fig, axes = plt.subplots(nrows, ncols, figsize=(1 * ncols, 1 * nrows))
+
+    # Plot ROIs in an ncols x nrows grid
+    colors = sns.color_palette()
+    for i, (img, ax) in enumerate(zip(roilist, axes.flatten())):
+        ax.imshow(img)
+        ax.set_xticks([])
+        ax.set_yticks([])
+        plt.setp(ax.spines.values(), color=colors[class_maxprob_list[i]], lw=4)
+        ax.text(
+            0,
+            1,
+            f"{class_maxprob_list[i]} {label_maxprob_list[i][:4].upper()}",
+            ha="left",
+            va="top",
+            fontsize=8,
+            transform=ax.transAxes,
+        )
+        if true_class and (label_maxprob_list[i] != true_class):
+            ax.text(
+                0.5,
+                0.5,
+                "X",
+                ha="center",
+                va="center",
+                fontsize=20,
+                color="red",
+                alpha=0.5,
+                transform=ax.transAxes,
+            )
+
+    # Hide non-used axes in the grid
+    for ax in axes.flatten()[len(roilist):]:
+        ax.set_visible(False)
+
+    fig.patch.set_linewidth(10)
+    fig.patch.set_edgecolor("k")
+
+    return fig, ax
+
+
+def classify_plot_class_rois(class_name, classifier, filelist):
+    """Classify single-object (ROI) images and plot images in a grid with best guess class.
+
+    Parameters
+    ----------
+    class_name: str
+        Name of class ROI files belong to (e.g. 'copepod')
+    classifier: pyopia.classify.Classify
+        PyOPIA classifier instance
+    filelist: list
+        List of single-object (ROI) files
+
+    Returns
+    -------
+    df_: pandas.DataFrame
+        Classification results for each image
+    """
+    # Load single-object images (ROIs)
+    roilist = [np.float64(plt.imread(f)) / 255.0 for f in filelist]
+
+    # Classify ROIs
+    df_ = classify_rois(roilist, classifier)
+    df_ = pyopia.statistics.add_best_guesses_to_stats(df_)
+
+    # Remove "_probability" from labels
+    df_ = df_.replace("probability_", "", regex=True)
+    df_.columns = df_.columns.str.replace("probability_", "", regex=True)
+
+    # Plot
+    fig, ax = plot_classified_rois(roilist, df_, true_class=class_name)
+
+    # Print classification info
+    num_correct = (df_["best guess"] == class_name).sum()
+    num_images = len(roilist)
+    frac_class = num_correct / num_images
+    print(
+        f"Correctly identified {class_name} was {100 * frac_class:.1f}% ({num_correct}/{num_images})"
+    )
+    fig.suptitle(
+        f"Class: {class_name} ({num_correct}/{num_images}, {100 * frac_class:.1f}%)"
+    )
+
+    return df_.style.format(precision=1, decimal=".")

{pyopia-2.9.3 → pyopia-2.10.0}/pyopia/tests/test_classify.py

@@ -1,7 +1,7 @@
-'''
+"""
 A high level test for the basic processing pipeline.
 
-'''
+"""
 
 from glob import glob
 import tempfile
@@ -21,48 +21,48 @@ import numpy as np
 import pyopia.instrument.silcam
 
 
-ACCURACY = 39
+ACCURACY = 60
 
 
 def test_match_to_database():
-    '''
+    """
     Basic check of classification prediction against the training database.
     Therefore, if correct positive matches are not high percentages, then something is wrong with the prediction.
 
     @todo include more advanced testing of the classification feks. assert values in a confusion matrix.
-    '''
+    """
 
     # location of the training data
     with tempfile.TemporaryDirectory() as tempdir:
         # location of the training data
-        database_path = os.path.join(tempdir, 'silcam_classification_database')
+        database_path = os.path.join(tempdir, "silcam_classification_database")
 
         exampledata.get_classifier_database_from_pysilcam_blob(database_path)
-        os.makedirs(os.path.join(tempdir, 'model'), exist_ok=True)
-        model_path = exampledata.get_example_model(os.path.join(tempdir, 'model'))
+        os.makedirs(os.path.join(tempdir, "model"), exist_ok=True)
+        model_path = exampledata.get_example_model(os.path.join(tempdir, "model"))
 
         # Load the trained tensorflow model and class names
         cl = pyopia.classify.Classify(model_path=model_path)
         class_labels = cl.class_labels
 
         # class_labels should match the training data
-        classes = sorted(glob(os.path.join(database_path, '*')))
+        classes = sorted(glob(os.path.join(database_path, "*")))
 
         # @todo write a quick check that classes and class_labels agree before doing the proper test.
 
         def correct_positives(category):
-            '''
+            """
             calculate the percentage positive matches for a given category
-            '''
-            print('Checking', category)
+            """
+            print("Checking", category)
             # list the files in this category of the training data
-            files = glob(os.path.join(database_path, category, '*.tiff'))
+            files = glob(os.path.join(database_path, category, "*.tiff"))
 
-            assert len(files) > 50, 'less then 50 files in test data.'
+            assert len(files) > 50, "less then 50 files in test data."
 
             # start a counter of incorrectly classified images
             failed = 0
-            time_limit = len(files) * 0.01
+            time_limit = len(files) * 0.02
             t1 = pd.Timestamp.now()
 
             # loop through the database images
@@ -83,7 +83,7 @@ def test_match_to_database():
 
             t2 = pd.Timestamp.now()
             td = t2 - t1
-            assert td < pd.to_timedelta(time_limit, 's'), 'Processing time too long.'
+            assert td < pd.to_timedelta(time_limit, "s"), "Processing time too long."
 
             return success
 
@@ -92,35 +92,37 @@ def test_match_to_database():
             name = os.path.split(cat)[-1]
             success = correct_positives(name)
             print(name, success)
-            assert success > ACCURACY, (name + ' was poorly classified at only ' + str(success) + 'percent.')
+            assert success > ACCURACY, (
+                name + " was poorly classified at only " + str(success) + "percent."
+            )
 
 
 def test_pipeline_classification():
-    '''Check that the pipeline doesn't change the outcome of the classification.
+    """Check that the pipeline doesn't change the outcome of the classification.
     Do this by putting rois of know classificion (which we know get correctly classified independintly),
     and then use the same model in a pipeline analysing the synthetic image.
-    '''
+    """
 
     with tempfile.TemporaryDirectory() as tempdir:
         # location of the training data
-        database_path = os.path.join(tempdir, 'silcam_classification_database')
+        database_path = os.path.join(tempdir, "silcam_classification_database")
 
         exampledata.get_classifier_database_from_pysilcam_blob(database_path)
-        os.makedirs('model', exist_ok=True)
-        model_path = exampledata.get_example_model('model')
+        os.makedirs("model", exist_ok=True)
+        model_path = exampledata.get_example_model("model")
 
         # Load the trained tensorflow model and class names
         cl = pyopia.classify.Classify(model_path=model_path)
 
         def get_good_roi(category):
-            '''
+            """
             calculate the percentage positive matches for a given category
-            '''
+            """
 
-            print('category', category)
+            print("category", category)
             # list the files in this category of the training data
-            files = sorted(glob(os.path.join(database_path, category, '*.tiff')))
-            print(len(files), 'files')
+            files = sorted(glob(os.path.join(database_path, category, "*.tiff")))
+            print(len(files), "files")
 
             found_match = 0
             # loop through the database images
@@ -136,16 +138,18 @@ def test_pipeline_classification():
 
                 # check if the highest score matches the correct category
                 if cl.class_labels[ind] == category:
-                    print('roi file', file)
+                    print("roi file", file)
                     return img, category
-            assert found_match == 1, f'classifier not finding matching particle for {category}'
+            assert found_match == 1, (
+                f"classifier not finding matching particle for {category}"
+            )
 
         canvas = np.ones((2048, 2448, 3), np.float64)
 
-        rc_shift = int(2048/len(cl.class_labels)/1.5)
+        rc_shift = int(2048 / len(cl.class_labels) / 1.5)
         rc = rc_shift
 
-        classes = sorted(glob(os.path.join(database_path, '*')))
+        classes = sorted(glob(os.path.join(database_path, "*")))
 
         categories = []
 
@@ -155,75 +159,96 @@ def test_pipeline_classification():
             categories.append(category)
             img_shape = np.shape(img)
             rc += rc_shift
-            canvas[rc:rc + img_shape[0], rc: rc + img_shape[1], :] = np.float64(img)
+            canvas[rc: rc + img_shape[0], rc: rc + img_shape[1], :] = np.float64(img)
 
-        settings = {'general': {'raw_files': None,
-                                'pixel_size': 24},
-                    'steps': {'note': 'non-standard pipeline.'}
-                    }
+        settings = {
+            "general": {"raw_files": None, "pixel_size": 24},
+            "steps": {"note": "non-standard pipeline."},
+        }
 
         # Initialise the pipeline class without running anything
-        MyPipeline = pyopia.pipeline.Pipeline(settings=settings, initial_steps='')
+        MyPipeline = pyopia.pipeline.Pipeline(settings=settings, initial_steps="")
 
         # Get the example trained model
         model_path = pyopia.exampledata.get_example_model(os.getcwd())
 
         # Add the classifier step description to settings (i.e. metadata)
-        MyPipeline.settings['steps'].update({'classifier':
-                                            {'pipeline_class': 'pyopia.classify.Classify',
-                                             'model_path': model_path}})
+        MyPipeline.settings["steps"].update(
+            {
+                "classifier": {
+                    "pipeline_class": "pyopia.classify.Classify",
+                    "model_path": model_path,
+                }
+            }
+        )
 
         # Execute the classifier step we defined above
-        MyPipeline.run_step('classifier')
+        MyPipeline.run_step("classifier")
         # This is the same as running:
         # MyPipeline.data['cl'] = pyopia.classify.Classify(model_path=model_path)
         # Note: the classifier step is special in that it's output is specifically data['cl'], rather than other new keys in data
 
-        MyPipeline.data['imraw'] = canvas
-        MyPipeline.data['timestamp'] = pd.Timestamp.now()
-        MyPipeline.data['filename'] = ''
+        MyPipeline.data["imraw"] = canvas
+        MyPipeline.data["timestamp"] = pd.Timestamp.now()
+        MyPipeline.data["filename"] = ""
 
         # Add the imageprep step description
-        MyPipeline.settings['steps'].update({'imageprep':
-                                            {'pipeline_class': 'pyopia.instrument.silcam.ImagePrep',
-                                             'image_level': 'imraw'}})
+        MyPipeline.settings["steps"].update(
+            {
+                "imageprep": {
+                    "pipeline_class": "pyopia.instrument.silcam.ImagePrep",
+                    "image_level": "imraw",
+                }
+            }
+        )
         # Run the step
-        MyPipeline.run_step('imageprep')
+        MyPipeline.run_step("imageprep")
         # This is the same as running:
         # ImagePrep = pyopia.instrument.silcam.ImagePrep(image_level='imraw')
         # MyPipeline.data = ImagePrep(MyPipeline.data)
 
         # Add the segmentation step description
-        MyPipeline.settings['steps'].update({'segmentation':
-                                            {'pipeline_class': 'pyopia.process.Segment',
-                                             'threshold': 1,
-                                             'segment_source': 'im_minimum'}})
+        MyPipeline.settings["steps"].update(
+            {
+                "segmentation": {
+                    "pipeline_class": "pyopia.process.Segment",
+                    "threshold": 1,
+                    "segment_source": "im_minimum",
+                }
+            }
+        )
         # Run the step
-        MyPipeline.run_step('segmentation')
+        MyPipeline.run_step("segmentation")
         # This is the same as running:
         # Segment = pyopia.process.Segment(threshold=settings['steps']['segmentation']['threshold'])
         # data = Segment(data)
 
         # Add the segmentation step description
-        MyPipeline.settings['steps'].update({'statextract':
-                                            {'pipeline_class': 'pyopia.process.CalculateStats',
-                                             'roi_source': 'imref'}}
-                                            )
+        MyPipeline.settings["steps"].update(
+            {
+                "statextract": {
+                    "pipeline_class": "pyopia.process.CalculateStats",
+                    "roi_source": "imref",
+                }
+            }
+        )
 
         # Run the step
-        MyPipeline.run_step('statextract')
+        MyPipeline.run_step("statextract")
         # This is the same as running:
         # CalculateStats = pyopia.process.CalculateStats()
         # data = CalculateStats(data)
 
-        stats = pyopia.statistics.add_best_guesses_to_stats(MyPipeline.data['stats'])
+        stats = pyopia.statistics.add_best_guesses_to_stats(MyPipeline.data["stats"])
 
-        out = [x[12:] for x in stats['best guess'].values]
+        out = [x[12:] for x in stats["best guess"].values]
 
-        print('classes input', categories)
-        print('classes measured', out)
+        print("classes input", categories)
+        print("classes measured", out)
 
-        assert categories == out, 'Classes returned from classifier do not match what was given to the pipeline'
+        assert categories == out, (
+            "Classes returned from classifier do not match what was given to the pipeline"
+        )
 
 
 if __name__ == "__main__":

pyopia-2.10.0/pyopia/tests/test_notebooks.py

@@ -0,0 +1,26 @@
+"""
+A high level test for executing the ipynb notebooks in the notebooks folder.
+
+Can only be run from top-level directory (i.e. with 'poetry run pytest -v')
+
+"""
+
+from nbconvert.preprocessors import ExecutePreprocessor
+import nbformat
+from pathlib import Path
+
+
+def test_notebooks():
+    notebooks = sorted(Path("notebooks/").rglob("*.ipynb"))
+    notebooks.append("docs/notebooks/background_correction.ipynb")
+    for notebook_filename in notebooks:
+        with open(notebook_filename, encoding="utf8") as f:
+            nb = nbformat.read(f, as_version=4)
+            ep = ExecutePreprocessor()
+            print("running", notebook_filename)
+            ep.preprocess(nb, {"metadata": {"path": "notebooks/"}})
+            print(notebook_filename, "complete")
+
+
+if __name__ == "__main__":
+    test_notebooks()

{pyopia-2.9.3 → pyopia-2.10.0}/pyproject.toml

@@ -17,7 +17,7 @@ keywords = [
 ]
 dependencies = [
     "flake8>=6.1.0,<7",
-    "numpy>=1.24.0,<2",
+    "numpy>=1.24.0",
     "scipy>=1.11.2,<2",
     "pytest>=7.2.0",
     "imageio>=2.31.3,<3",
@@ -47,13 +47,15 @@ dependencies = [
     "nbconvert>=7.16.4,<8",
     "h5netcdf>= 1.3.0",
     "scikit-image>=0.24.0,<0.25",
+    "click<8.2.0",
+    "seaborn>=0.13.2",
 ]
 
 [project.optional-dependencies]
 
 classification = [
-    "tensorflow>=2.16.2,<3",
-    "keras==3.5.0",
+    "tensorflow>=2.19.0",
+    "keras==3.9.1",
 ]
 
 [project.urls]

pyopia-2.9.3/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.9.3"

pyopia-2.9.3/pyopia/exampledata.py

@@ -1,143 +0,0 @@
-import urllib.request
-import zipfile
-import os
-import gdown
-
-import logging
-logger = logging.getLogger()
-
-
-def get_classifier_database_from_pysilcam_blob(download_directory='./'):
-    '''Downloads a specified filename from the pysilcam.blob into the working dir. if it doesn't already exist
-
-    only works for known filenames that are on this blob
-
-    Parameters
-    ----------
-    filename : string
-        known filename on the blob
-
-    '''
-    if os.path.exists(os.path.join(download_directory)):
-        logger.info(download_directory, 'already exists. Returning nothing')
-        return download_directory
-    os.makedirs(download_directory, exist_ok=False)
-    url = 'https://pysilcam.blob.core.windows.net/test-data/silcam_database.zip'
-    logger.info('Downloading....')
-    urllib.request.urlretrieve(url, download_directory + '/silcam_database.zip')
-    logger.info('Unzipping....')
-    with zipfile.ZipFile(os.path.join(download_directory, 'silcam_database.zip'), 'r') as zipit:
-        zipit.extractall(os.path.join(download_directory, '../'))
-    logger.info('Removing zip file')
-    os.remove(os.path.join(download_directory, 'silcam_database.zip'))
-    logger.info('Done.')
-    return download_directory
-
-
-def get_file_from_pysilcam_blob(filename, download_directory='./'):
-    '''Downloads a specified filename from the pysilcam.blob into the working dir. if it doesn't already exist
-
-    only works for known filenames that are on this blob
-
-    Parameters
-    ----------
-    filename : string
-        known filename on the blob
-
-    '''
-    if os.path.exists(os.path.join(download_directory, filename)):
-        return filename
-    url = 'https://pysilcam.blob.core.windows.net/test-data/' + filename
-    urllib.request.urlretrieve(url, os.path.join(download_directory, filename))
-    return download_directory
-
-
-def get_example_silc_image(download_directory='./'):
-    '''calls `get_file_from_pysilcam_blob` for a silcam iamge
-
-    Returns
-    -------
-    string
-        filename
-    '''
-    filename = 'D20181101T142731.838206.silc'
-    if os.path.isfile(filename):
-        logger.info('Example image already exists. Skipping download.')
-        return filename
-    get_file_from_pysilcam_blob(filename, download_directory)
-    return filename
-
-
-def get_example_model(download_directory='./'):
-    '''Downloads and unzips an example trained CNN model from the pysilcam.blob
-    into the working dir. if it doesn't already exist.
-
-    Returns
-    -------
-    string
-        model_filename
-    '''
-    model_filename = 'silcam-classification_database_20240822-200-20240829T091048-best-epoch.keras'
-    if os.path.exists(model_filename):
-        return model_filename
-    # -- Download and unzip the model --#
-    url = 'https://pysilcam.blob.core.windows.net/test-data/silcam-classification_database_20240822-200-20240829T091048.zip'
-    zip_filename = 'silcam-classification_database_20240822-200-20240829T091048.zip'
-    urllib.request.urlretrieve(url, os.path.join(download_directory, zip_filename))
-    with zipfile.ZipFile(os.path.join(download_directory, zip_filename), 'r') as zipit:
-        zipit.extractall(download_directory)
-    return os.path.join(download_directory, model_filename.replace('-best-epoch.keras', ''), model_filename)
-
-
-def get_example_hologram_and_background(download_directory='./'):
-    '''calls `get_file_from_pysilcam_blob` for a raw hologram, and its associated background image.
-
-    Returns
-    -------
-    string
-        holo_filename
-
-    string
-        holo_background_filename
-    '''
-    holo_filename = '001-2082.pgm'
-    holo_background_filename = 'imbg-' + holo_filename
-    get_file_from_pysilcam_blob(holo_filename, download_directory)
-    get_file_from_pysilcam_blob(holo_background_filename, download_directory)
-
-    holo_filename = os.path.join(download_directory, holo_filename)
-    holo_background_filename = os.path.join(download_directory, holo_background_filename)
-
-    return holo_filename, holo_background_filename
-
-
-def get_folder_from_holo_repository(foldername="holo_test_data_01", existsok=False):
-    '''Downloads a specified folder from the holo testing repository into the working dir. if it doesn't already exist
-
-    only works for known folders that are on the GoogleDrive repository
-    by default will download a known-good folder. Additional elif statements can be added to implement additional folders.
-
-    Parameters
-    ----------
-    foldername : string
-        known filename on the blob
-    existsok : (bool, optional)
-        if True, then don't download if the specified folder already exists, defaults to False
-
-    '''
-    if foldername == "holo_test_data_01":
-        url = 'https://drive.google.com/drive/folders/1yNatOaKdWwYQp-5WVEDItoibr-k0lGsP?usp=share_link'
-
-    elif foldername == "holo_test_data_02":
-        url = "https://drive.google.com/drive/folders/1E5iNSyfeKcVMLVe4PNEwF2Q2mo3WVjF5?usp=share_link"
-
-    else:
-        foldername == "holo_test_data_01"
-        url = 'https://drive.google.com/drive/folders/1yNatOaKdWwYQp-5WVEDItoibr-k0lGsP?usp=share_link'
-
-    if os.path.exists(foldername) and existsok:
-        logger.info(foldername + ' already exists. Skipping download.')
-        return foldername
-
-    gdown.download_folder(url, quiet=True, use_cookies=False)
-    return foldername

pyopia-2.9.3/pyopia/plotting.py

@@ -1,51 +0,0 @@
-# -*- coding: utf-8 -*-
-'''
-Particle plotting functionality for standardised figures
-e.g. image presentation, size distributions, montages etc.
-'''
-
-import matplotlib.pyplot as plt
-import numpy as np
-
-
-def show_image(image, pixel_size):
-    '''Plots a scaled figure (in mm) of an image
-
-    Parameters
-    ----------
-    image : float
-        Image (usually a corrected image, such as im_corrected)
-    pixel_size : float
-        the pixel size (um) of the imaging system used
-    '''
-    r, c = np.shape(image[:, :, 0])
-
-    plt.imshow(image,
-               extent=[0, c * pixel_size / 1000, 0, r * pixel_size / 1000],
-               interpolation='nearest')
-    plt.xlabel('mm')
-    plt.ylabel('mm')
-
-    return
-
-
-def montage_plot(montage, pixel_size):
-    '''
-    Plots a SilCam particle montage with a 1mm scale reference
-
-    Parameters
-    ----------
-    montage : uint8
-        a montage created with scpp.make_montage
-    pixel_size : float
-        the pixel size (um) of the imaging system used
-    '''
-    msize = np.shape(montage)[0]
-    ex = pixel_size * np.float64(msize) / 1000.
-
-    ax = plt.gca()
-    ax.imshow(montage, extent=[0, ex, 0, ex], cmap='grey')
-    ax.set_xticks([1, 2], [])
-    ax.set_xticklabels(['    1mm', ''])
-    ax.set_yticks([], [])
-    ax.xaxis.set_ticks_position('bottom')

pyopia-2.9.3/pyopia/tests/test_notebooks.py

@@ -1,22 +0,0 @@
-'''
-A high level test for executing the ipynb notebooks in the notebooks folder.
-
-Can only be run from top-level directory (i.e. with 'poetry run pytest -v')
-
-'''
-
-from nbconvert.preprocessors import ExecutePreprocessor
-import nbformat
-from glob import glob
-
-
-def test_notebooks():
-    notebooks = sorted(glob('notebooks/*.ipynb'))
-    notebooks.append('docs/notebooks/background_correction.ipynb')
-    for notebook_filename in notebooks:
-        with open(notebook_filename) as f:
-            nb = nbformat.read(f, as_version=4)
-            ep = ExecutePreprocessor()
-            print('running', notebook_filename)
-            ep.preprocess(nb, {'metadata': {'path': 'notebooks/'}})
-            print(notebook_filename, 'complete')