cellfinder 1.4.1__py3-none-any.whl → 1.9.0__py3-none-any.whl

cellfinder/core/detect/filters/volume/ball_filter.py

@@ -78,11 +78,11 @@ class BallFilter:
     ----------
     plane_height, plane_width : int
         Height/width of the planes.
-    ball_xy_size : int
-        Diameter of the spherical kernel in the x/y dimensions.
-    ball_z_size : int
-        Diameter of the spherical kernel in the z dimension.
-        Equal to the number of planes stacked to filter
+    ball_xy_size : float
+        Diameter of the spherical kernel (in microns) in the x/y dimensions.
+    ball_z_size : float
+        Diameter of the spherical kernel in the z dimension in microns.
+        Determines the number of planes stacked to filter
         the central plane of the stack.
     overlap_fraction : float
         The fraction of pixels within the spherical kernel that

cellfinder/core/detect/filters/volume/structure_detection.py

@@ -222,6 +222,11 @@ class CellDetector:
                         neighbour_ids[2] = previous_plane[y, x]
 
                     if is_new_structure(neighbour_ids):
+                        if self.next_structure_id > self.soma_centre_value:
+                            raise ValueError(
+                                "label overflow: number of connected "
+                                "components exceeds label capacity"
+                            )
                         neighbour_ids[0] = self.next_structure_id
                         self.next_structure_id += 1
                     struct_id = self.add(x, y, self.z, neighbour_ids)

cellfinder/core/detect/filters/volume/structure_splitting.py

@@ -1,3 +1,4 @@
+from copy import copy
 from typing import List, Tuple, Type
 
 import numpy as np
@@ -65,8 +66,7 @@ def coords_to_volume(
     relative_zs = np.array((zs - z_min + ball_radius), dtype=np.int64)
 
     # set each point as the center with a value of threshold
-    for rel_x, rel_y, rel_z in zip(relative_xs, relative_ys, relative_zs):
-        volume[rel_x, rel_y, rel_z] = threshold_value
+    volume[relative_xs, relative_ys, relative_zs] = threshold_value
 
     volume = volume.swapaxes(0, 2)
     return torch.from_numpy(volume)
@@ -224,6 +224,7 @@ def split_cells(
             where M is the number of individual cells and each centre is
             represented by its x, y, and z coordinates.
     """
+    settings = copy(settings)
     # these points are in x, y, z order columnwise, in absolute pixels
     orig_centre = get_structure_centre(cell_points)
 

cellfinder/core/detect/filters/volume/volume_filter.py

@@ -140,7 +140,7 @@ class VolumeFilter:
             tensor = torch.empty(
                 (batch_size, *self.settings.plane_shape),
                 dtype=torch_dtype,
-                pin_memory=not cpu,
+                pin_memory=not cpu and self.settings.pin_memory,
                 device="cpu",
             )
 

cellfinder/core/download/download.py

@@ -6,6 +6,7 @@ import pooch
 from brainglobe_utils.general.config import get_config_obj
 
 from cellfinder import DEFAULT_CELLFINDER_DIRECTORY
+from cellfinder.core import logger
 from cellfinder.core.tools.source_files import (
     default_configuration_path,
     user_specific_configuration_path,
@@ -74,7 +75,7 @@ def amend_user_configuration(new_model_path=None) -> None:
     new_model_path : Path, optional
         The path to the new model configuration.
     """
-    print("(Over-)writing custom user configuration")
+    logger.info("(Over-)writing custom user configuration")
 
     original_config = default_configuration_path()
     new_config = user_specific_configuration_path()

cellfinder/core/main.py

@@ -1,34 +1,33 @@
 import os
 from typing import Callable, List, Optional, Tuple
 
-import numpy as np
 from brainglobe_utils.cells.cells import Cell
 
-from cellfinder.core import logger
+from cellfinder.core import logger, types
 from cellfinder.core.download.download import model_type
-from cellfinder.core.train.train_yml import depth_type
+from cellfinder.core.train.train_yaml import depth_type
 
 
 def main(
-    signal_array: np.ndarray,
-    background_array: np.ndarray,
-    voxel_sizes: Tuple[int, int, int],
+    signal_array: types.array,
+    background_array: types.array,
+    voxel_sizes: Tuple[float, float, float],
     start_plane: int = 0,
     end_plane: int = -1,
     trained_model: Optional[os.PathLike] = None,
     model_weights: Optional[os.PathLike] = None,
     model: model_type = "resnet50_tv",
-    batch_size: int = 64,
+    classification_batch_size: int = 64,
     n_free_cpus: int = 2,
-    network_voxel_sizes: Tuple[int, int, int] = (5, 1, 1),
-    soma_diameter: int = 16,
-    ball_xy_size: int = 6,
-    ball_z_size: int = 15,
+    network_voxel_sizes: Tuple[float, float, float] = (5, 1, 1),
+    soma_diameter: float = 16,
+    ball_xy_size: float = 6,
+    ball_z_size: float = 15,
     ball_overlap_fraction: float = 0.6,
     log_sigma_size: float = 0.2,
     n_sds_above_mean_thresh: float = 10,
     soma_spread_factor: float = 1.4,
-    max_cluster_size: int = 100000,
+    max_cluster_size: float = 100000,
     cube_width: int = 50,
     cube_height: int = 50,
     cube_depth: int = 20,
@@ -36,8 +35,15 @@ def main(
     skip_detection: bool = False,
     skip_classification: bool = False,
     detected_cells: List[Cell] = None,
-    classification_batch_size: Optional[int] = None,
-    classification_torch_device: str = "cpu",
+    detection_batch_size: Optional[int] = None,
+    torch_device: Optional[str] = None,
+    pin_memory: bool = False,
+    split_ball_xy_size: float = 6,
+    split_ball_z_size: float = 15,
+    split_ball_overlap_fraction: float = 0.8,
+    n_splitting_iter: int = 10,
+    n_sds_above_mean_tiled_thresh: float = 10,
+    tiled_thresh_tile_size: float | None = None,
     *,
     detect_callback: Optional[Callable[[int], None]] = None,
     classify_callback: Optional[Callable[[int], None]] = None,
@@ -46,6 +52,125 @@ def main(
     """
     Parameters
     ----------
+    signal_array : numpy.ndarray or dask array
+        3D array representing the signal data in z, y, x order.
+    background_array : numpy.ndarray or dask array
+        3D array representing the signal data in z, y, x order.
+    voxel_sizes : 3-tuple of floats
+        Size of your voxels in the z, y, and x dimensions (microns).
+    start_plane : int
+        First plane index to process (inclusive, to process a subset of the
+        data).
+    end_plane : int
+        Last plane index to process (exclusive, to process a subset of the
+        data).
+    trained_model : Optional[Path]
+        Trained model file path (home directory (default) -> pretrained
+        weights).
+    model_weights : Optional[Path]
+        Model weights path (home directory (default) -> pretrained
+        weights).
+    model: str
+        Type of model to use. Defaults to `"resnet50_tv"`.
+    classification_batch_size : int
+        How many potential cells to classify at one time. The GPU/CPU
+        memory must be able to contain at once this many data cubes for
+        the models. For performance-critical applications, tune to maximize
+        memory usage without running out. Check your GPU/CPU memory to verify
+        it's not full.
+    n_free_cpus : int
+        How many CPU cores to leave free.
+    network_voxel_sizes : 3-tuple of floats
+        Size of the pre-trained network's voxels (microns) in the z, y, and x
+        dimensions.
+    soma_diameter : float
+        The expected in-plane (xy) soma diameter (microns).
+    ball_xy_size : float
+        3d filter's in-plane (xy) filter ball size (microns).
+    ball_z_size : float
+        3d filter's axial (z) filter ball size (microns).
+    ball_overlap_fraction : float
+        3d filter's fraction of the ball filter needed to be filled by
+        foreground voxels, centered on a voxel, to retain the voxel.
+    log_sigma_size : float
+        Gaussian filter width (as a fraction of soma diameter) used during
+        2d in-plane Laplacian of Gaussian filtering.
+    n_sds_above_mean_thresh : float
+        Per-plane intensity threshold (the number of standard deviations
+        above the mean) of the filtered 2d planes used to mark pixels as
+        foreground or background.
+    soma_spread_factor : float
+        Cell spread factor for determining the largest cell volume before
+        splitting up cell clusters. Structures with spherical volume of
+        diameter `soma_spread_factor * soma_diameter` or less will not be
+        split.
+    max_cluster_size : float
+        Largest detected cell cluster (in cubic um) where splitting
+        should be attempted. Clusters above this size will be labeled
+        as artifacts.
+    cube_width: int
+        The width of the data cube centered on the cell used for
+        classification. Defaults to `50`.
+    cube_height: int
+        The height of the data cube centered on the cell used for
+        classification. Defaults to `50`.
+    cube_depth: int
+        The depth of the data cube centered on the cell used for
+        classification. Defaults to `20`.
+    network_depth: str
+        The network depth to use during classification. Defaults to `"50"`.
+    skip_detection : bool
+        If selected, the detection step is skipped and instead we get the
+        detected cells from the cell layer below (from a previous
+        detection run or import).
+    skip_classification : bool
+        If selected, the classification step is skipped and all cells from
+        the detection stage are added.
+    detected_cells: Optional list of Cell objects.
+        If specified, the cells to use during classification.
+    detection_batch_size: int
+        The number of planes of the original data volume to process at
+        once. The GPU/CPU memory must be able to contain this many planes
+        for all the filters. For performance-critical applications, tune
+        to maximize memory usage without running out. Check your GPU/CPU
+        memory to verify it's not full.
+    torch_device : str, optional
+        The device on which to run the computation. If not specified (None),
+        "cuda" will be used if a GPU is available, otherwise "cpu".
+        You can also manually specify "cuda" or "cpu".
+    pin_memory: bool
+        Pins data to be sent to the GPU to the CPU memory. This allows faster
+        GPU data speeds, but can only be used if the data used by the GPU can
+        stay in the CPU RAM while the GPU uses it. I.e. there's enough RAM.
+        Otherwise, if there's a risk of the RAM being paged, it shouldn't be
+        used. Defaults to False.
+    split_ball_xy_size: float
+        Similar to `ball_xy_size`, except the value to use for the 3d
+        filter during cluster splitting.
+    split_ball_z_size: float
+        Similar to `ball_z_size`, except the value to use for the 3d filter
+        during cluster splitting.
+    split_ball_overlap_fraction: float
+        Similar to `ball_overlap_fraction`, except the value to use for the
+        3d filter during cluster splitting.
+    n_splitting_iter: int
+        The number of iterations to run the 3d filtering on a cluster. Each
+        iteration reduces the cluster size by the voxels not retained in
+        the previous iteration.
+    n_sds_above_mean_tiled_thresh : float
+        Per-plane, per-tile intensity threshold (the number of standard
+        deviations above the mean) for the filtered 2d planes used to mark
+        pixels as foreground or background. When used, (tile size is not zero)
+        a pixel is marked as foreground if its intensity is above both the
+        per-plane and per-tile threshold. I.e. it's above the set number of
+        standard deviations of the per-plane average and of the per-plane
+        per-tile average for the tile that contains it.
+    tiled_thresh_tile_size : float
+        The tile size used to tile the x, y plane to calculate the local
+        average intensity for the tiled threshold. The value is multiplied
+        by soma diameter (i.e. 1 means one soma diameter). If zero or None, the
+        tiled threshold is disabled and only the per-plane threshold is used.
+        Tiling is done with 50% overlap when striding.
     detect_callback : Callable[int], optional
         Called every time a plane has finished being processed during the
         detection stage. Called with the plane number that has finished.
@@ -63,22 +188,29 @@ def main(
         logger.info("Detecting cell candidates")
 
         points = detect.main(
-            signal_array,
-            start_plane,
-            end_plane,
-            voxel_sizes,
-            soma_diameter,
-            max_cluster_size,
-            ball_xy_size,
-            ball_z_size,
-            ball_overlap_fraction,
-            soma_spread_factor,
-            n_free_cpus,
-            log_sigma_size,
-            n_sds_above_mean_thresh,
-            batch_size=classification_batch_size,
-            torch_device=classification_torch_device,
+            signal_array=signal_array,
+            start_plane=start_plane,
+            end_plane=end_plane,
+            voxel_sizes=voxel_sizes,
+            soma_diameter=soma_diameter,
+            max_cluster_size=max_cluster_size,
+            ball_xy_size=ball_xy_size,
+            ball_z_size=ball_z_size,
+            ball_overlap_fraction=ball_overlap_fraction,
+            soma_spread_factor=soma_spread_factor,
+            n_free_cpus=n_free_cpus,
+            log_sigma_size=log_sigma_size,
+            n_sds_above_mean_thresh=n_sds_above_mean_thresh,
+            n_sds_above_mean_tiled_thresh=n_sds_above_mean_tiled_thresh,
+            tiled_thresh_tile_size=tiled_thresh_tile_size,
+            batch_size=detection_batch_size,
+            torch_device=torch_device,
+            pin_memory=pin_memory,
             callback=detect_callback,
+            split_ball_z_size=split_ball_z_size,
+            split_ball_xy_size=split_ball_xy_size,
+            split_ball_overlap_fraction=split_ball_overlap_fraction,
+            n_splitting_iter=n_splitting_iter,
         )
 
         if detect_finished_callback is not None:
@@ -101,7 +233,7 @@ def main(
                 n_free_cpus,
                 voxel_sizes,
                 network_voxel_sizes,
-                batch_size,
+                classification_batch_size,
                 cube_height,
                 cube_width,
                 cube_depth,

cellfinder/core/tools/threading.py

@@ -15,6 +15,7 @@ Typical example::
 
     from cellfinder.core.tools.threading import ThreadWithException, \\
         EOFSignal, ProcessWithException
+    from cellfinder.core import logger
     import torch
 
 
@@ -63,7 +64,7 @@ Typical example::
                     # thread exited for whatever reason (not exception)
                     break
 
-                print(f"Thread processed tensor {i}")
+                logger.debug(f"Thread processed tensor {i}")
         finally:
             # whatever happens, make sure thread is told to finish so it
             # doesn't get stuck
@@ -248,8 +249,8 @@ class ExceptionWithQueueMixIn:
             ...         # do something with the msg
             ...         pass
             ... except ExecutionFailure as e:
-            ...     print(f"got exception {type(e.__cause__)}")
-            ...     print(f"with message {e.__cause__.args[0]}")
+            ...     logger.error(f"got exception {type(e.__cause__)}")
+            ...     logger.error(f"with message {e.__cause__.args[0]}")
         """
         msg, value = self.from_thread_queue.get(block=True, timeout=timeout)
         if msg == "eof":

cellfinder/core/tools/tools.py

@@ -261,7 +261,7 @@ def random_bool(likelihood: Optional[float] = None) -> bool:
     if likelihood is None:
         return bool(getrandbits(1))
     else:
-        if uniform(0, 1) > likelihood:
+        if uniform(0, 1) < likelihood:
             return True
         else:
             return False

cellfinder/core/train/{train_yml.py → train_yaml.py}

@@ -3,9 +3,6 @@ main
 ===============
 
 Trains a network based on a yaml file specifying cubes of cells/non cells.
-
-N.B imports are within functions to prevent tensorflow being imported before
-it's warnings are silenced
 """
 
 import os
@@ -29,12 +26,16 @@ from brainglobe_utils.general.system import (
 from brainglobe_utils.IO.cells import find_relevant_tiffs
 from brainglobe_utils.IO.yaml import read_yaml_section
 from fancylog import fancylog
+from keras.callbacks import CSVLogger, ModelCheckpoint, TensorBoard
 from sklearn.model_selection import train_test_split
 
-import cellfinder.core as program_for_log
+import cellfinder.core as package_for_log
 from cellfinder.core import logger
+from cellfinder.core.classify.cube_generator import CubeGeneratorFromDisk
 from cellfinder.core.classify.resnet import layer_type
+from cellfinder.core.classify.tools import get_model, make_lists
 from cellfinder.core.download.download import DEFAULT_DOWNLOAD_DIRECTORY
+from cellfinder.core.tools.prep import prep_model_weights
 
 depth_type = Literal["18", "34", "50", "101", "152"]
 
@@ -268,7 +269,7 @@ def cli():
 
     fancylog.start_logging(
         args.output_dir,
-        program_for_log,
+        package=package_for_log,
         variables=[args],
         log_header="CELLFINDER TRAINING LOG",
     )
@@ -316,16 +317,6 @@ def run(
     save_progress=False,
     epochs=100,
 ):
-    from keras.callbacks import (
-        CSVLogger,
-        ModelCheckpoint,
-        TensorBoard,
-    )
-
-    from cellfinder.core.classify.cube_generator import CubeGeneratorFromDisk
-    from cellfinder.core.classify.tools import get_model, make_lists
-    from cellfinder.core.tools.prep import prep_model_weights
-
     start_time = datetime.now()
 
     ensure_directory_exists(output_dir)

cellfinder/napari/curation.py

@@ -8,11 +8,11 @@ from brainglobe_napari_io.cellfinder.utils import convert_layer_to_cells
 from brainglobe_utils.cells.cells import Cell
 from brainglobe_utils.general.system import delete_directory_contents
 from brainglobe_utils.IO.yaml import save_yaml
-from brainglobe_utils.qtpy.dialog import display_warning
-from brainglobe_utils.qtpy.interaction import add_button, add_combobox
 from magicgui.widgets import ProgressBar
 from napari.qt.threading import thread_worker
 from napari.utils.notifications import show_info
+from qt_niu.dialog import display_info, display_warning
+from qt_niu.interaction import add_button, add_combobox
 from qtpy import QtCore
 from qtpy.QtWidgets import (
     QComboBox,
@@ -95,6 +95,22 @@ class CurationWidget(QWidget):
                 self.training_data_non_cell_choice, self.point_layer_names
             )
 
+        @self.viewer.layers.events.removed.connect
+        def _remove_selection_layers(event: QtCore.QEvent):
+            """
+            Set internal background, signal, training data cell,
+            and training data non-cell layers to None when they
+            are removed from the napari viewer GUI.
+            """
+            if event.value == self.signal_layer:
+                self.signal_layer = None
+            if event.value == self.background_layer:
+                self.background_layer = None
+            if event.value == self.training_data_cell_layer:
+                self.training_data_cell_layer = None
+            if event.value == self.training_data_non_cell_layer:
+                self.training_data_non_cell_layer = None
+
     @staticmethod
     def _update_combobox_options(combobox: QComboBox, options_list: List[str]):
         original_text = combobox.currentText()
@@ -212,8 +228,8 @@ class CurationWidget(QWidget):
         self.layout.addWidget(self.load_data_panel, row, column, 1, 1)
 
     def setup_keybindings(self):
-        self.viewer.bind_key("c", self.mark_as_cell)
-        self.viewer.bind_key("x", self.mark_as_non_cell)
+        self.viewer.bind_key("c", self.mark_as_cell, overwrite=True)
+        self.viewer.bind_key("x", self.mark_as_non_cell, overwrite=True)
 
     def set_signal_image(self):
         """
@@ -406,7 +422,7 @@ class CurationWidget(QWidget):
             self.update_status_label("Ready")
 
     def __prep_directories_for_save(self):
-        self.yaml_filename = self.output_directory / "training.yml"
+        self.yaml_filename = self.output_directory / "training.yaml"
         self.cell_cube_dir = self.output_directory / "cells"
         self.no_cell_cube_dir = self.output_directory / "non_cells"
 
@@ -479,26 +495,61 @@ class CurationWidget(QWidget):
             return False
 
     def check_training_data_exists(self) -> bool:
-        if not (
-            self.training_data_cell_layer or self.training_data_non_cell_layer
-        ):
-            show_info(
-                "No training data layers have been added. "
-                "Please add a layer and annotate some points.",
+        """
+        Checks that
+        - both training data layers exists
+        - at least one of them is not empty.
+
+        Will display a popup dialog and return False if these conditions
+        are not both fulfilled.
+
+        Will show a notification if only one layer is non-empty, but this
+        is considered valid.
+
+        Returns
+        -------
+        bool
+            True if both training layers exists and at least one
+            of them contains some data. False otherwise.
+        """
+        both_training_layers_exist = (
+            self.training_data_cell_layer and self.training_data_non_cell_layer
+        )
+
+        if not both_training_layers_exist:
+            display_info(
+                self,
+                "No training data layers have been added.",
+                "Please add layers for both cells and non-cells,"
+                "and annotate some points.",
             )
             return False
-        else:
-            if (
-                len(self.training_data_cell_layer.data) > 0
-                or len(self.training_data_non_cell_layer.data) > 0
-            ):
-                return True
-            else:
+
+        at_least_one_training_layer_contains_data = (
+            len(self.training_data_cell_layer.data) > 0
+            or len(self.training_data_non_cell_layer.data) > 0
+        )
+
+        both_training_layers_contain_data = (
+            len(self.training_data_cell_layer.data) > 0
+            and len(self.training_data_non_cell_layer.data) > 0
+        )
+
+        if at_least_one_training_layer_contains_data:
+            if not both_training_layers_contain_data:
                 show_info(
-                    "No training data points have been added. "
-                    "Please annotate some points.",
+                    "One of the training layers is empty. This is OK, but"
+                    "For optimal (re-)training ensure you have roughly equal "
+                    "number of points in each of your training points layers."
                 )
-                return False
+            return True
+        else:
+            display_info(
+                self,
+                "No training data points have been added.",
+                "Please annotate points in the training data layers.",
+            )
+            return False
 
     def get_output_directory(self):
         """