stouputils 1.14.0__py3-none-any.whl

stouputils/data_science/config/set.py

@@ -0,0 +1,125 @@
+""" Configuration file for the project. """
+
+# Imports
+import os
+from typing import Literal
+
+from stouputils.decorators import LogLevels
+from stouputils.io import get_root_path
+
+# Environment variables
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "9"         # Suppress TensorFlow logging
+os.environ["GRPC_VERBOSITY"] = "ERROR"           # Suppress gRPC logging
+
+
+# Configuration class
+class DataScienceConfig:
+	""" Configuration class for the project. """
+
+	# Common
+	SEED: int = 42
+	""" Seed for the random number generator. """
+
+	ERROR_LOG: LogLevels = LogLevels.WARNING_TRACEBACK
+	""" Log level for errors for all functions. """
+
+	AUGMENTED_FILE_SUFFIX: str = "_aug_"
+	""" Suffix for augmented files, e.g. 'image_008_aug_1.png'. """
+
+	AUGMENTED_DIRECTORY_PREFIX: str = "aug_"
+	""" Prefix for augmented directories, e.g. 'data/hip_implant' -> 'data/aug_hip_implant'. """
+
+	PREPROCESSED_DIRECTORY_SUFFIX: str = "_preprocessed"
+	""" Suffix for preprocessed directories, e.g. 'data/hip_implant' -> 'data/hip_implant_preprocessed'. """
+
+
+	# Directories
+	ROOT: str = get_root_path(__file__, go_up=3)
+	""" Root directory of the project. """
+
+	MLFLOW_FOLDER: str = f"{ROOT}/mlruns"
+	""" Folder containing the mlflow data. """
+	MLFLOW_URI: str = f"file://{MLFLOW_FOLDER}"
+	""" URI to the mlflow data. """
+
+	DATA_FOLDER: str = f"{ROOT}/data"
+	""" Folder containing all the data (e.g. subfolders containing images). """
+
+	TEMP_FOLDER: str = f"{ROOT}/temp"
+	""" Folder containing temporary files (e.g. models checkpoints, plots, etc.). """
+
+	LOGS_FOLDER: str = f"{ROOT}/logs"
+	""" Folder containing the logs. """
+
+	TENSORBOARD_FOLDER: str = f"{ROOT}/tensorboard"
+	""" Folder containing the tensorboard logs. """
+
+
+	# Behaviours
+	TEST_SIZE: float = 0.2
+	""" Size of the test set by default (0.2 means 80% training, 20% test). """
+
+	VALIDATION_SIZE: float = 0.2
+	""" Size of the validation set by default (0.2 means 80% training, 20% validation). """
+
+	# Machine learning
+	SAVE_MODEL: bool = False
+	""" If the model should be saved in the mlflow folder using mlflow.*.save_model. """
+
+	DO_SALIENCY_AND_GRADCAM: bool = True
+	""" If the saliency and gradcam should be done during the run. """
+
+	DO_LEARNING_RATE_FINDER: Literal[0, 1, 2] = 1
+	""" If the learning rate finder should be done during the run.
+	0: no, 1: only plot, 2: plot and use value for the remaining run
+	"""
+
+	DO_UNFREEZE_FINDER: Literal[0, 1, 2] = 0
+	""" If the unfreeze finder should be done during the run.
+	0: no, 1: only plot, 2: plot and use value for the remaining run
+	"""
+
+	DO_FIT_IN_SUBPROCESS: bool = True
+	""" If the model should be fitted in a subprocess.
+	Is memory efficient, and more stable. Turn it off only if you are having issues.
+
+	Note: This allow a program to make lots of runs without getting killed by the OS for using too much resources.
+	(e.g. LeaveOneOut Cross Validation, Grid Search, etc.)
+	"""
+
+	MIXED_PRECISION_POLICY: Literal["mixed_float16", "mixed_bfloat16", "float32"] = "mixed_float16"
+	""" Mixed precision policy to use. Turn back to "float32" if you are having issues with a specific model or metrics.
+	See: https://www.tensorflow.org/guide/mixed_precision
+	"""
+
+	TENSORFLOW_DEVICE: str = "/gpu:1"
+	""" TensorFlow device to use. """
+
+
+
+	@classmethod
+	def update_root(cls, new_root: str) -> None:
+		""" Update the root directory and recalculate all dependent paths. 
+		
+		Args:
+			new_root: The new root directory path.
+		"""
+		cls.ROOT = new_root
+		
+		# Update all paths that depend on ROOT
+		cls.MLFLOW_FOLDER = f"{cls.ROOT}/mlruns"
+		cls.MLFLOW_URI = f"file://{cls.MLFLOW_FOLDER}"
+		cls.DATA_FOLDER = f"{cls.ROOT}/data"
+		cls.TEMP_FOLDER = f"{cls.ROOT}/temp"
+		cls.LOGS_FOLDER = f"{cls.ROOT}/logs"
+		cls.TENSORBOARD_FOLDER = f"{cls.ROOT}/tensorboard"
+		
+		# Fix MLFLOW_URI for Windows by adding a missing slash
+		if os.name == "nt":
+			cls.MLFLOW_URI = cls.MLFLOW_URI.replace("file:", "file:/")
+
+
+# Fix MLFLOW_URI for Windows by adding a missing slash
+if os.name == "nt":
+	DataScienceConfig.MLFLOW_URI = DataScienceConfig.MLFLOW_URI.replace("file:", "file:/")
+

stouputils/data_science/data_processing/image/__init__.py

@@ -0,0 +1,66 @@
+
+# Imports
+from .auto_contrast import auto_contrast_image
+from .axis_flip import flip_image
+from .bias_field_correction import bias_field_correction_image
+from .binary_threshold import binary_threshold_image
+from .blur import blur_image
+from .brightness import brightness_image
+from .canny import canny_image
+from .clahe import clahe_image
+from .contrast import contrast_image
+from .curvature_flow_filter import curvature_flow_filter_image
+from .denoise import (
+    adaptive_denoise_image,
+    bilateral_denoise_image,
+    nlm_denoise_image,
+    tv_denoise_image,
+    wavelet_denoise_image,
+)
+from .invert import invert_image
+from .laplacian import laplacian_image
+from .median_blur import median_blur_image
+from .noise import noise_image
+from .normalize import normalize_image
+from .random_erase import random_erase_image
+from .resize import resize_image
+from .rotation import rotate_image
+from .salt_pepper import salt_pepper_image
+from .sharpening import sharpen_image
+from .shearing import shear_image
+from .threshold import threshold_image
+from .translation import translate_image
+from .zoom import zoom_image
+
+__all__ = [
+    "adaptive_denoise_image",
+    "auto_contrast_image",
+    "bias_field_correction_image",
+    "bilateral_denoise_image",
+    "binary_threshold_image",
+    "blur_image",
+    "brightness_image",
+    "canny_image",
+    "clahe_image",
+    "contrast_image",
+    "curvature_flow_filter_image",
+    "flip_image",
+    "invert_image",
+    "laplacian_image",
+    "median_blur_image",
+    "nlm_denoise_image",
+    "noise_image",
+    "normalize_image",
+    "random_erase_image",
+    "resize_image",
+    "rotate_image",
+    "salt_pepper_image",
+    "sharpen_image",
+    "shear_image",
+    "threshold_image",
+    "translate_image",
+    "tv_denoise_image",
+    "wavelet_denoise_image",
+    "zoom_image",
+]
+

stouputils/data_science/data_processing/image/auto_contrast.py

@@ -0,0 +1,79 @@
+
+# pyright: reportUnusedImport=false
+# ruff: noqa: F401
+
+# Imports
+from .common import Any, NDArray, check_image, cv2, np
+
+
+# Functions
+def auto_contrast_image(image: NDArray[Any], ignore_dtype: bool = False) -> NDArray[Any]:
+	""" Adjust the contrast of an image.
+
+	Args:
+		image         (NDArray[Any]):  Image to adjust contrast
+		ignore_dtype  (bool):        Ignore the dtype check
+	Returns:
+		NDArray[Any]: Image with adjusted contrast
+
+	>>> ## Basic tests
+	>>> image = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.uint8)
+	>>> adjusted = auto_contrast_image(image)
+	>>> adjusted.tolist()
+	[[0, 36, 73], [109, 146, 182], [219, 255, 255]]
+	>>> adjusted.shape == image.shape
+	True
+	>>> adjusted.dtype == image.dtype
+	True
+
+	>>> ## Test invalid inputs
+	>>> auto_contrast_image("not an image")
+	Traceback (most recent call last):
+		...
+	AssertionError: Image must be a numpy array
+	"""
+	# Check input data
+	check_image(image, ignore_dtype=ignore_dtype)
+
+	# Perform histogram clipping
+	clip_hist_percent: float = 1.0
+
+	# Calculate the histogram of the image
+	hist: NDArray[Any] = cv2.calcHist([image], [0], None, [256], [0, 256])
+
+	# Create an accumulator list to store the cumulative histogram
+	accumulator: list[float] = []
+	accumulator.append(hist[0])
+	for i in range(1, 256):
+		accumulator.append(accumulator[i - 1] + hist[i])
+
+	# Find the maximum value in the accumulator
+	max_value: float = accumulator[-1]
+
+	# Calculate the clipping threshold
+	clip_hist_percent = clip_hist_percent * (max_value / 100.0)
+	clip_hist_percent = clip_hist_percent / 2.0
+
+	# Find the minimum and maximum gray levels after clipping
+	min_gray: int = 0
+	while accumulator[min_gray] < clip_hist_percent:
+		min_gray = min_gray + 1
+	max_gray: int = 256 - 1
+	while (max_gray >= 0 and accumulator[max_gray] >= (max_value - clip_hist_percent)):
+		max_gray = max_gray - 1
+
+	# Calculate the input range after clipping
+	input_range: int = max_gray - min_gray
+
+	# If the input range is 0, return the original image
+	if input_range == 0:
+		return image
+
+	# Calculate the scaling factors for contrast adjustment
+	alpha: float = (256 - 1) / input_range
+	beta: float = -min_gray * alpha
+
+	# Apply the contrast adjustment
+	adjusted: NDArray[Any] = cv2.convertScaleAbs(image, alpha=alpha, beta=beta)
+	return adjusted
+

stouputils/data_science/data_processing/image/axis_flip.py

@@ -0,0 +1,58 @@
+
+# pyright: reportUnusedImport=false
+# ruff: noqa: F401
+
+# Imports
+from typing import Literal
+
+from .common import Any, NDArray, check_image, cv2, np
+
+
+# Functions
+def flip_image(
+	image: NDArray[Any], axis: Literal["horizontal", "vertical", "both"], ignore_dtype: bool = True
+) -> NDArray[Any]:
+	""" Flip an image along specified axis
+
+	Args:
+		image         (NDArray[Any]):     Image to flip
+		axis          (str):            Axis along which to flip ("horizontal" or "vertical" or "both")
+		ignore_dtype  (bool):           Ignore the dtype check
+	Returns:
+		NDArray[Any]: Flipped image
+
+	>>> ## Basic tests
+	>>> image = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
+	>>> flip_image(image, "horizontal").tolist()
+	[[3, 2, 1], [6, 5, 4], [9, 8, 7]]
+
+	>>> flip_image(image, "vertical").tolist()
+	[[7, 8, 9], [4, 5, 6], [1, 2, 3]]
+
+	>>> flip_image(image, "both").tolist()
+	[[9, 8, 7], [6, 5, 4], [3, 2, 1]]
+
+	>>> ## Test invalid inputs
+	>>> flip_image(image, "diagonal")
+	Traceback (most recent call last):
+	AssertionError: axis must be either 'horizontal' or 'vertical' or 'both', got 'diagonal'
+
+	>>> flip_image("not an image", "horizontal")
+	Traceback (most recent call last):
+		...
+	AssertionError: Image must be a numpy array
+	"""
+	# Check input data
+	check_image(image, ignore_dtype=ignore_dtype)
+	assert axis in ("horizontal", "vertical", "both"), (
+		f"axis must be either 'horizontal' or 'vertical' or 'both', got '{axis}'"
+	)
+
+	# Apply the flip
+	if axis == "horizontal":
+		return cv2.flip(image, 1)	# 1 for horizontal flip
+	elif axis == "vertical":
+		return cv2.flip(image, 0)	# 0 for vertical flip
+	else:
+		return cv2.flip(image, -1)	# -1 for both flips
+

stouputils/data_science/data_processing/image/bias_field_correction.py

@@ -0,0 +1,74 @@
+
+# pyright: reportUnknownMemberType=false
+# pyright: reportUnknownArgumentType=false
+# pyright: reportUnknownVariableType=false
+
+# Imports
+import SimpleITK as Sitk
+
+from .common import Any, NDArray, check_image, np
+
+
+# Functions
+def bias_field_correction_image(image: NDArray[Any], ignore_dtype: bool = False) -> NDArray[Any]:
+	""" Apply a bias field correction to an image. (N4 Filter)
+
+	Args:
+		image         (NDArray[Any]):  Image to apply the bias field correction (can't be 8-bit unsigned integer)
+		ignore_dtype  (bool):        Ignore the dtype check
+	Returns:
+		NDArray[Any]: Image with the curvature flow filter applied
+
+	>>> ## Basic tests
+	>>> image = np.random.randint(0, 255, size=(10,10), dtype=np.uint8) / 255
+	>>> corrected = bias_field_correction_image(image)
+	>>> corrected.shape == image.shape
+	True
+	>>> corrected.dtype == np.float64
+	True
+
+	>>> ## Test invalid inputs
+	>>> bias_field_correction_image("not an image")
+	Traceback (most recent call last):
+		...
+	AssertionError: Image must be a numpy array
+	"""
+	# Check input data
+	check_image(image, ignore_dtype=ignore_dtype)
+
+	# If the image is 3D, convert to grayscale first
+	if image.ndim == 3:
+		image = np.mean(image, axis=-1)
+
+	# Convert numpy array to SimpleITK image
+	image_sitk: Sitk.Image = Sitk.GetImageFromArray(image)
+
+	# Create binary mask of the head region
+	transformed: Sitk.Image = Sitk.RescaleIntensity(image_sitk)  # Normalize intensities
+	transformed = Sitk.LiThreshold(transformed, 0, 1)  # Apply Li thresholding
+	head_mask: Sitk.Image = transformed
+
+	# Downsample images to speed up bias field estimation
+	shrink_factor: int = 4  # Reduce image size by factor of 4
+	input_image: Sitk.Image = Sitk.Shrink(
+		image_sitk,
+		[shrink_factor] * image_sitk.GetDimension()  # Apply shrink factor to all dimensions
+	)
+	mask_image: Sitk.Image = Sitk.Shrink(
+		head_mask,
+		[shrink_factor] * image_sitk.GetDimension()
+	)
+
+	# Apply N4 bias field correction
+	corrector = Sitk.N4BiasFieldCorrectionImageFilter()
+	corrector.Execute(input_image, mask_image)
+
+	# Get estimated bias field and apply correction
+	log_bias_field: Sitk.Image = Sitk.Cast(
+		corrector.GetLogBiasFieldAsImage(image_sitk), Sitk.sitkFloat64
+	)
+	corrected_image_full_resolution: Sitk.Image = image_sitk / Sitk.Exp(log_bias_field)
+
+	# Convert back to numpy array and return
+	return Sitk.GetArrayFromImage(corrected_image_full_resolution)
+

stouputils/data_science/data_processing/image/binary_threshold.py

@@ -0,0 +1,73 @@
+
+# pyright: reportUnusedImport=false
+# ruff: noqa: F401
+
+# Imports
+from .common import Any, NDArray, check_image, cv2, np
+
+
+# Functions
+def binary_threshold_image(image: NDArray[Any], threshold: float, ignore_dtype: bool = False) -> NDArray[Any]:
+	""" Apply binary threshold to an image.
+
+	Args:
+		image         (NDArray[Any]):  Image to threshold
+		threshold     (float):       Threshold value (between 0 and 1)
+		ignore_dtype  (bool):        Ignore the dtype check
+	Returns:
+		NDArray[Any]: Thresholded binary image
+
+	>>> ## Basic tests
+	>>> image = np.array([[100, 150, 200], [50, 125, 175], [25, 75, 225]])
+	>>> binary_threshold_image(image.astype(np.uint8), 0.5).tolist()
+	[[0, 255, 255], [0, 0, 255], [0, 0, 255]]
+
+	>>> np.random.seed(42)
+	>>> img = np.random.randint(0, 256, (4,4), dtype=np.uint8)
+	>>> thresholded = binary_threshold_image(img, 0.7)
+	>>> set(np.unique(thresholded).tolist()) <= {0, 255}  # Should only contain 0 and 255
+	True
+
+	>>> rgb = np.random.randint(0, 256, (3,3,3), dtype=np.uint8)
+	>>> thresh_rgb = binary_threshold_image(rgb, 0.5)
+	>>> thresh_rgb.shape == rgb.shape
+	True
+	>>> set(np.unique(thresh_rgb).tolist()) <= {0, 255}
+	True
+
+	>>> ## Test invalid inputs
+	>>> binary_threshold_image("not an image", 0.5)
+	Traceback (most recent call last):
+		...
+	AssertionError: Image must be a numpy array
+
+	>>> binary_threshold_image(image.astype(np.uint8), "0.5")
+	Traceback (most recent call last):
+		...
+	AssertionError: threshold must be a number, got <class 'str'>
+
+	>>> binary_threshold_image(image.astype(np.uint8), 1.5)
+	Traceback (most recent call last):
+		...
+	AssertionError: threshold must be between 0 and 1, got 1.5
+	"""
+	# Check input data
+	check_image(image, ignore_dtype=ignore_dtype)
+	assert isinstance(threshold, float | int), f"threshold must be a number, got {type(threshold)}"
+	assert 0 <= threshold <= 1, f"threshold must be between 0 and 1, got {threshold}"
+
+	# Convert threshold from 0-1 range to 0-255 range
+	threshold_value: int = int(threshold * 255)
+
+	# Apply threshold
+	if len(image.shape) == 2:
+		# Grayscale image
+		binary: NDArray[Any] = cv2.threshold(image, threshold_value, 255, cv2.THRESH_BINARY)[1]
+	else:
+		# Color image - convert to grayscale first, then back to color
+		gray: NDArray[Any] = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+		binary: NDArray[Any] = cv2.threshold(gray, threshold_value, 255, cv2.THRESH_BINARY)[1]
+		binary = cv2.cvtColor(binary, cv2.COLOR_GRAY2BGR)
+
+	return binary
+

stouputils/data_science/data_processing/image/blur.py

@@ -0,0 +1,59 @@
+
+# pyright: reportUnusedImport=false
+# ruff: noqa: F401
+
+# Imports
+from .common import Any, NDArray, check_image, cv2, np
+
+
+# Functions
+def blur_image(image: NDArray[Any], blur_strength: float, ignore_dtype: bool = False) -> NDArray[Any]:
+	""" Apply Gaussian blur to an image.
+
+	Args:
+		image         (NDArray[Any]):  Image to blur
+		blur_strength (float):       Strength of the blur
+		ignore_dtype  (bool):        Ignore the dtype check
+	Returns:
+		NDArray[Any]: Blurred image
+
+	>>> ## Basic tests
+	>>> image = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
+	>>> blurred = blur_image(image.astype(np.uint8), 1.5)
+	>>> blurred.shape == image.shape
+	True
+
+	>>> img = np.zeros((5,5), dtype=np.uint8)
+	>>> img[2,2] = 255  # Single bright pixel
+	>>> blurred = blur_image(img, 1.0)
+	>>> bool(blurred[2,2] < 255)  # Center should be blurred
+	True
+
+	>>> rgb = np.full((3,3,3), 128, dtype=np.uint8)
+	>>> blurred_rgb = blur_image(rgb, 1.0)
+	>>> blurred_rgb.shape == (3,3,3)
+	True
+
+	>>> ## Test invalid inputs
+	>>> blur_image("not an image", 1.5)
+	Traceback (most recent call last):
+		...
+	AssertionError: Image must be a numpy array
+
+	>>> blur_image(image.astype(np.uint8), "1.5")
+	Traceback (most recent call last):
+		...
+	AssertionError: blur_strength must be a number, got <class 'str'>
+	"""
+	# Check input data
+	check_image(image, ignore_dtype=ignore_dtype)
+	assert isinstance(blur_strength, float | int), f"blur_strength must be a number, got {type(blur_strength)}"
+
+	# Apply Gaussian blur
+	kernel_size: int = max(3, int(blur_strength * 2) + 1)
+	if kernel_size % 2 == 0:
+		kernel_size += 1
+	blurred_image: NDArray[Any] = cv2.GaussianBlur(image, (kernel_size, kernel_size), 0)
+
+	return blurred_image
+

stouputils/data_science/data_processing/image/brightness.py

@@ -0,0 +1,54 @@
+
+# pyright: reportUnusedImport=false
+# ruff: noqa: F401
+
+# Imports
+from .common import Any, NDArray, check_image, cv2, np
+
+
+# Functions
+def brightness_image(image: NDArray[Any], brightness_factor: float, ignore_dtype: bool = False) -> NDArray[Any]:
+	""" Adjust the brightness of an image.
+
+	Args:
+		image             (NDArray[Any]):  Image to adjust brightness
+		brightness_factor (float):       Brightness adjustment factor
+		ignore_dtype      (bool):        Ignore the dtype check
+	Returns:
+		NDArray[Any]: Image with adjusted brightness
+
+	>>> ## Basic tests
+	>>> image = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
+	>>> brightened = brightness_image(image.astype(np.uint8), 1.5)
+	>>> brightened.shape == image.shape
+	True
+
+	>>> img = np.full((3,3), 100, dtype=np.uint8)
+	>>> bright = brightness_image(img, 2.0)
+	>>> dark = brightness_image(img, 0.5)
+	>>> bool(np.mean(bright) > np.mean(img) > np.mean(dark))
+	True
+
+	>>> rgb = np.full((3,3,3), 128, dtype=np.uint8)
+	>>> bright_rgb = brightness_image(rgb, 1.5)
+	>>> bright_rgb.shape == (3,3,3)
+	True
+
+	>>> ## Test invalid inputs
+	>>> brightness_image("not an image", 1.5)
+	Traceback (most recent call last):
+		...
+	AssertionError: Image must be a numpy array
+
+	>>> brightness_image(image.astype(np.uint8), "1.5")
+	Traceback (most recent call last):
+		...
+	AssertionError: brightness_factor must be a number, got <class 'str'>
+	"""
+	# Check input data
+	check_image(image, ignore_dtype=ignore_dtype)
+	assert isinstance(brightness_factor, float | int), f"brightness_factor must be a number, got {type(brightness_factor)}"
+
+	# Apply brightness adjustment
+	return cv2.convertScaleAbs(image, alpha=brightness_factor, beta=0)
+