stouputils 1.3.10__tar.gz → 1.3.12__tar.gz

{stouputils-1.3.10 → stouputils-1.3.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stouputils
-Version: 1.3.10
+Version: 1.3.12
 Summary: Stouputils is a collection of utility modules designed to simplify and enhance the development process. It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers, and many more.
 Project-URL: Homepage, https://github.com/Stoupy51/stouputils
 Project-URL: Issues, https://github.com/Stoupy51/stouputils/issues

{stouputils-1.3.10 → stouputils-1.3.12}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "stouputils"
-version = "1.3.10"
+version = "1.3.12"
 description = "Stouputils is a collection of utility modules designed to simplify and enhance the development process. It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers, and many more."
 readme = "README.md"
 requires-python = ">=3.10"

{stouputils-1.3.10 → stouputils-1.3.12}/stouputils/continuous_delivery/github.py

@@ -247,7 +247,7 @@ def generate_changelog(
 ) -> str:
 	""" Generate changelog from commits. They must follow the conventional commits convention.
 
-	Convention format: <type>: <description>
+	Convention format: <type>: <description> or <type>(<sub-category>): <description>
 
 	Args:
 		commits				(list[dict]):	List of commits to generate changelog from
@@ -261,7 +261,7 @@ def generate_changelog(
 		https://www.conventionalcommits.org/en/v1.0.0/
 	"""
 	# Initialize the commit groups
-	commit_groups: dict[str, list[tuple[str, str]]] = {}
+	commit_groups: dict[str, list[tuple[str, str, str | None]]] = {}
 
 	# Iterate over the commits
 	for commit in commits:
@@ -270,17 +270,27 @@ def generate_changelog(
 
 		# If the message contains a colon, split the message into a type and a description
 		if ":" in message:
-			commit_type, desc = message.split(":", 1)
-
-			# Clean the type, ex: 'feat(hand)/refactor(feet)' -> 'feat'
-			commit_type = commit_type.split('(')[0].split('/')[0]
+			commit_type_part, desc = message.split(":", 1)
+
+			# Extract sub-category if present (e.g., 'feat(Project)' -> 'feat', 'Project')
+			sub_category: str|None = None
+			if "(" in commit_type_part and ")" in commit_type_part:
+				# Extract the base type (before parentheses)
+				commit_type: str = commit_type_part.split('(')[0].split('/')[0]
+				# Extract the sub-category (between parentheses)
+				sub_category = commit_type_part.split('(')[1].split(')')[0]
+			else:
+				# No sub-category, just clean the type
+				commit_type: str = commit_type_part.split('/')[0]
+
+			# Clean the type to only keep letters
 			commit_type = "".join(c for c in commit_type.lower().strip() if c in "abcdefghijklmnopqrstuvwxyz")
 			commit_type = COMMIT_TYPES.get(commit_type, commit_type.title())
 
 			# Add the commit to the commit groups
 			if commit_type not in commit_groups:
 				commit_groups[commit_type] = []
-			commit_groups[commit_type].append((desc.strip(), sha))
+			commit_groups[commit_type].append((desc.strip(), sha, sub_category))
 
 	# Initialize the changelog
 	changelog: str = "## Changelog\n\n"
@@ -289,9 +299,32 @@ def generate_changelog(
 	for commit_type in sorted(commit_groups.keys()):
 		changelog += f"### {commit_type}\n"
 
-		# Reverse the list to display the most recent commits in last
-		for desc, sha in commit_groups[commit_type][::-1]:
-			changelog += f"- {desc} ([{sha[:7]}](https://github.com/{owner}/{project_name}/commit/{sha}))\n"
+		# Group commits by sub-category
+		sub_category_groups: dict[str|None, list[tuple[str, str, str|None]]] = {}
+		for desc, sha, sub_category in commit_groups[commit_type]:
+			if sub_category not in sub_category_groups:
+				sub_category_groups[sub_category] = []
+			sub_category_groups[sub_category].append((desc, sha, sub_category))
+
+		# Sort sub-categories (None comes first, then alphabetical)
+		sorted_sub_categories = sorted(
+			sub_category_groups.keys(),
+			key=lambda x: (x is None, x or "")
+		)
+
+		# Iterate over sub-categories
+		for sub_category in sorted_sub_categories:
+
+			# Add commits for this sub-category
+			for desc, sha, _ in reversed(sub_category_groups[sub_category]):
+
+				# Prepend sub-category to description if present
+				if sub_category:
+					formatted_desc = f"[{sub_category.replace('_', ' ').title()}] {desc}"
+				else:
+					formatted_desc = desc
+				changelog += f"- {formatted_desc} ([{sha[:7]}](https://github.com/{owner}/{project_name}/commit/{sha}))\n"
+
 		changelog += "\n"
 
 	# Add the full changelog link if there is a latest tag and return the changelog

{stouputils-1.3.10 → stouputils-1.3.12}/stouputils/data_science/models/model_interface.py

@@ -1,11 +1,53 @@
 """ Base implementation for machine learning models with common functionality.
 Provides shared infrastructure for model training, evaluation, and MLflow integration.
 
-Implements core workflow methods:
+Implements comprehensive workflow methods and features:
 
+Core Training & Evaluation:
 - Full training/evaluation pipeline (routine_full)
-- Transfer learning weight management
-- MLflow experiment tracking integration
+- K-fold cross-validation with stratified splitting
+- Transfer learning weight management (ImageNet, custom datasets)
+- Model prediction and evaluation with comprehensive metrics
+
+Hyperparameter Optimization:
+- Learning Rate Finder with automatic best LR detection
+- Unfreeze Percentage Finder for fine-tuning optimization
+- Class weight balancing for imbalanced datasets
+- Learning rate warmup and scheduling (ReduceLROnPlateau)
+
+Advanced Training Features:
+- Early stopping with configurable patience
+- Model checkpointing with delay options
+- Additional training data integration (bypasses CV splitting)
+- Multi-processing support for memory management
+- Automatic retry mechanisms with error handling
+
+MLflow Integration:
+- Complete experiment tracking and logging
+- Parameter logging (training, optimizer, callback parameters)
+- Metric logging with averages and standard deviations
+- Model artifact saving and versioning
+- Training history visualization and plotting
+
+Model Architecture Support:
+- Keras/TensorFlow and PyTorch compatibility
+- Automatic layer counting and fine-tuning
+- Configurable unfreeze percentages for transfer learning
+- Memory leak prevention with subprocess training
+
+Evaluation & Visualization:
+- ROC and PR curve generation
+- Comprehensive metric calculation (Sensitivity, Specificity, AUC, etc.)
+- Training history plotting and analysis
+- Saliency maps and GradCAM visualization (single sample)
+- Cross-validation results aggregation
+
+Configuration & Utilities:
+- Extensive parameter override system
+- Verbosity control throughout pipeline
+- Temporary directory management for artifacts
+- Garbage collection and memory optimization
+- Error logging and handling with retry mechanisms
 """
 # pyright: reportUnknownMemberType=false
 # pyright: reportUnknownArgumentType=false
@@ -16,6 +58,7 @@ from __future__ import annotations
 import gc
 import multiprocessing
 import multiprocessing.queues
+import time
 from collections.abc import Generator, Iterable
 from tempfile import TemporaryDirectory
 from typing import Any
@@ -27,7 +70,7 @@ from numpy.typing import NDArray
 from sklearn.utils import class_weight
 
 from ...decorators import handle_error, measure_time
-from ...print import progress, debug, info
+from ...print import progress, debug, info, warning
 from ...ctx import Muffle, MeasureTime
 from ...io import clean_path
 
@@ -741,17 +784,25 @@ class ModelInterface(AbstractModel):
 		Returns:
 			float: The best learning rate found.
 		"""
-		if DataScienceConfig.DO_FIT_IN_SUBPROCESS:
-			queue: multiprocessing.queues.Queue[dict[str, Any]] = multiprocessing.Queue()
-			process: multiprocessing.Process = multiprocessing.Process(
-				target=self._find_best_learning_rate_subprocess,
-				kwargs={"dataset": dataset, "queue": queue, "verbose": verbose}
-			)
-			process.start()
-			process.join()
-			results: dict[str, Any] = queue.get(timeout=60)
-		else:
-			results: dict[str, Any] = self._find_best_learning_rate_subprocess(dataset, verbose=verbose)
+		results: dict[str, Any] = {}
+		for try_count in range(10):
+			try:
+				if DataScienceConfig.DO_FIT_IN_SUBPROCESS:
+					queue: multiprocessing.queues.Queue[dict[str, Any]] = multiprocessing.Queue()
+					process: multiprocessing.Process = multiprocessing.Process(
+						target=self._find_best_learning_rate_subprocess,
+						kwargs={"dataset": dataset, "queue": queue, "verbose": verbose}
+					)
+					process.start()
+					process.join()
+					results = queue.get(timeout=60)
+				else:
+					results = self._find_best_learning_rate_subprocess(dataset, verbose=verbose)
+				if results:
+					break
+			except Exception as e:
+				warning(f"Error finding best learning rate: {e}\nRetrying in 60 seconds ({try_count + 1}/10)...")
+				time.sleep(60)
 
 		# Plot the learning rate vs loss and find the best learning rate
 		return MetricUtils.find_best_x_and_plot(
@@ -777,17 +828,25 @@ class ModelInterface(AbstractModel):
 		Returns:
 			float: The best unfreeze percentage found.
 		"""
-		if DataScienceConfig.DO_FIT_IN_SUBPROCESS:
-			queue: multiprocessing.queues.Queue[dict[str, Any]] = multiprocessing.Queue()
-			process: multiprocessing.Process = multiprocessing.Process(
-				target=self._find_best_unfreeze_percentage_subprocess,
-				kwargs={"dataset": dataset, "queue": queue, "verbose": verbose}
-			)
-			process.start()
-			process.join()
-			results: dict[str, Any] = queue.get(timeout=60)
-		else:
-			results: dict[str, Any] = self._find_best_unfreeze_percentage_subprocess(dataset, verbose=verbose)
+		results: dict[str, Any] = {}
+		for try_count in range(10):
+			try:
+				if DataScienceConfig.DO_FIT_IN_SUBPROCESS:
+					queue: multiprocessing.queues.Queue[dict[str, Any]] = multiprocessing.Queue()
+					process: multiprocessing.Process = multiprocessing.Process(
+						target=self._find_best_unfreeze_percentage_subprocess,
+						kwargs={"dataset": dataset, "queue": queue, "verbose": verbose}
+					)
+					process.start()
+					process.join()
+					results = queue.get(timeout=60)
+				else:
+					results = self._find_best_unfreeze_percentage_subprocess(dataset, verbose=verbose)
+				if results:
+					break
+			except Exception as e:
+				warning(f"Error finding best unfreeze percentage: {e}\nRetrying in 60 seconds ({try_count + 1}/10)...")
+				time.sleep(60)
 
 		# Plot the unfreeze percentage vs loss and find the best unfreeze percentage
 		return MetricUtils.find_best_x_and_plot(
@@ -823,20 +882,26 @@ class ModelInterface(AbstractModel):
 			temp_dir = TemporaryDirectory()
 
 		# Create and run the process
-		if DataScienceConfig.DO_FIT_IN_SUBPROCESS and fold_number > 0:
-			queue: multiprocessing.queues.Queue[dict[str, Any]] = multiprocessing.Queue()
-			process: multiprocessing.Process = multiprocessing.Process(
-				target=self._train_subprocess,
-				args=(dataset, checkpoint_path, temp_dir),
-				kwargs={"queue": queue, "verbose": verbose}
-			)
-			process.start()
-			process.join()
-			return_values: dict[str, Any] = queue.get(timeout=60)
-		else:
-			return_values: dict[str, Any] = self._train_subprocess(
-				dataset, checkpoint_path, temp_dir, verbose=verbose
-			)
+		return_values: dict[str, Any] = {}
+		for try_count in range(10):
+			try:
+				if DataScienceConfig.DO_FIT_IN_SUBPROCESS and fold_number > 0:
+					queue: multiprocessing.queues.Queue[dict[str, Any]] = multiprocessing.Queue()
+					process: multiprocessing.Process = multiprocessing.Process(
+						target=self._train_subprocess,
+						args=(dataset, checkpoint_path, temp_dir),
+						kwargs={"queue": queue, "verbose": verbose}
+					)
+					process.start()
+					process.join()
+					return_values = queue.get(timeout=60)
+				else:
+					return_values = self._train_subprocess(dataset, checkpoint_path, temp_dir, verbose=verbose)
+				if return_values:
+					break
+			except Exception as e:
+				warning(f"Error during _train_fold: {e}\nRetrying in 60 seconds ({try_count + 1}/10)...")
+				time.sleep(60)
 		history: dict[str, Any] = return_values["history"]
 		eval_results: dict[str, Any] = return_values["eval_results"]
 		predictions: NDArray[Any] = return_values["predictions"]