stouputils 1.14.0__py3-none-any.whl

stouputils/collections.py

@@ -0,0 +1,244 @@
+"""
+This module provides utilities for collection manipulation:
+
+- unique_list: Remove duplicates from a list while preserving order using object id, hash or str
+- sort_dict_keys: Sort dictionary keys using a given order list (ascending or descending)
+- upsert_in_dataframe: Insert or update a row in a Polars DataFrame based on primary keys
+- array_to_disk: Easily handle large numpy arrays on disk using zarr for efficient storage and access.
+
+.. image:: https://raw.githubusercontent.com/Stoupy51/stouputils/refs/heads/main/assets/collections_module.gif
+  :alt: stouputils collections examples
+"""
+
+# Imports
+import atexit
+import os
+import shutil
+import tempfile
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any, Literal, TypeVar
+
+# Lazy imports for typing
+if TYPE_CHECKING:
+	import numpy as np
+	import polars as pl
+	import zarr  # pyright: ignore[reportMissingTypeStubs]
+	from numpy.typing import NDArray
+
+# Typing
+T = TypeVar("T")
+
+# Functions
+def unique_list[T](list_to_clean: Iterable[T], method: Literal["id", "hash", "str"] = "str") -> list[T]:
+	""" Remove duplicates from the list while keeping the order using ids (default) or hash or str
+
+	Args:
+		list_to_clean	(Iterable[T]):					The list to clean
+		method			(Literal["id", "hash", "str"]):	The method to use to identify duplicates
+	Returns:
+		list[T]: The cleaned list
+
+	Examples:
+		>>> unique_list([1, 2, 3, 2, 1], method="id")
+		[1, 2, 3]
+
+		>>> s1 = {1, 2, 3}
+		>>> s2 = {2, 3, 4}
+		>>> s3 = {1, 2, 3}
+		>>> unique_list([s1, s2, s1, s1, s3, s2, s3], method="id")
+		[{1, 2, 3}, {2, 3, 4}, {1, 2, 3}]
+
+		>>> s1 = {1, 2, 3}
+		>>> s2 = {2, 3, 4}
+		>>> s3 = {1, 2, 3}
+		>>> unique_list([s1, s2, s1, s1, s3, s2, s3], method="str")
+		[{1, 2, 3}, {2, 3, 4}]
+	"""
+	# Initialize the seen ids set and the result list
+	seen: set[int | str] = set()
+	result: list[T] = []
+
+	# Iterate over each item in the list
+	for item in list_to_clean:
+		if method == "id":
+			item_identifier = id(item)
+		elif method == "hash":
+			item_identifier = hash(item)
+		elif method == "str":
+			item_identifier = str(item)
+		else:
+			raise ValueError(f"Invalid method: {method}")
+
+		# If the item id is not in the seen ids set, add it to the seen ids set and append the item to the result list
+		if item_identifier not in seen:
+			seen.add(item_identifier)
+			result.append(item)
+
+	# Return the cleaned list
+	return result
+
+def sort_dict_keys[T](dictionary: dict[T, Any], order: list[T], reverse: bool = False) -> dict[T, Any]:
+	""" Sort dictionary keys using a given order list (reverse optional)
+
+	Args:
+		dictionary	(dict[T, Any]):	The dictionary to sort
+		order		(list[T]):		The order list
+		reverse		(bool):			Whether to sort in reverse order (given to sorted function which behaves differently than order.reverse())
+	Returns:
+		dict[T, Any]: The sorted dictionary
+
+	Examples:
+		>>> sort_dict_keys({'b': 2, 'a': 1, 'c': 3}, order=["a", "b", "c"])
+		{'a': 1, 'b': 2, 'c': 3}
+
+		>>> sort_dict_keys({'b': 2, 'a': 1, 'c': 3}, order=["a", "b", "c"], reverse=True)
+		{'c': 3, 'b': 2, 'a': 1}
+
+		>>> sort_dict_keys({'b': 2, 'a': 1, 'c': 3, 'd': 4}, order=["c", "b"])
+		{'c': 3, 'b': 2, 'a': 1, 'd': 4}
+	"""
+	return dict(sorted(dictionary.items(), key=lambda x: order.index(x[0]) if x[0] in order else len(order), reverse=reverse))
+
+def upsert_in_dataframe(
+	df: "pl.DataFrame",
+	new_entry: dict[str, Any],
+	primary_keys: dict[str, Any] | None = None
+) -> "pl.DataFrame":
+	""" Insert or update a row in the Polars DataFrame based on primary keys.
+
+	Args:
+		df				(pl.DataFrame):		The Polars DataFrame to update.
+		new_entry		(dict[str, Any]):	The new entry to insert or update.
+		primary_keys	(dict[str, Any]):	The primary keys to identify the row (default: empty).
+	Returns:
+		pl.DataFrame: The updated Polars DataFrame.
+	"""
+	# Imports
+	import polars as pl
+
+	# Create new DataFrame if file doesn't exist or is invalid
+	if df.is_empty():
+		return pl.DataFrame([new_entry])
+
+	# If no primary keys provided, return DataFrame with new entry appended
+	if not primary_keys:
+		new_row_df = pl.DataFrame([new_entry])
+		return pl.concat([df, new_row_df], how="diagonal_relaxed")
+
+	# Build mask based on primary keys
+	mask: pl.Expr = pl.lit(True)
+	for key, value in primary_keys.items():
+		if key in df.columns:
+			mask = mask & (df[key] == value)
+		else:
+			# Primary key column doesn't exist, so no match possible
+			mask = pl.lit(False)
+			break
+
+	# Insert or update row based on primary keys
+	if df.select(mask).to_series().any():
+		# Update existing row
+		for key, value in new_entry.items():
+			if key in df.columns:
+				df = df.with_columns(pl.when(mask).then(pl.lit(value)).otherwise(pl.col(key)).alias(key))
+			else:
+				# Add new column if it doesn't exist
+				df = df.with_columns(pl.when(mask).then(pl.lit(value)).otherwise(None).alias(key))
+		return df
+	else:
+		# Insert new row
+		new_row_df = pl.DataFrame([new_entry])
+		return pl.concat([df, new_row_df], how="diagonal_relaxed")
+
+def array_to_disk(
+	data: "NDArray[Any] | zarr.Array",
+	delete_input: bool = True,
+	more_data: "NDArray[Any] | zarr.Array | None" = None
+) -> tuple["zarr.Array", str, int]:
+	""" Easily handle large numpy arrays on disk using zarr for efficient storage and access.
+
+	Zarr provides a simpler and more efficient alternative to np.memmap with better compression
+	and chunking capabilities.
+
+	Args:
+		data			(NDArray | zarr.Array):	The data to save/load as a zarr array
+		delete_input	(bool):	Whether to delete the input data after creating the zarr array
+		more_data		(NDArray | zarr.Array | None): Additional data to append to the zarr array
+	Returns:
+		tuple[zarr.Array, str, int]: The zarr array, the directory path, and the total size in bytes
+
+	Examples:
+		>>> import numpy as np
+		>>> data = np.random.rand(1000, 1000)
+		>>> zarr_array = array_to_disk(data)[0]
+		>>> zarr_array.shape
+		(1000, 1000)
+
+		>>> more_data = np.random.rand(500, 1000)
+		>>> longer_array, dir_path, total_size = array_to_disk(zarr_array, more_data=more_data)
+	"""
+	def dir_size(directory: str) -> int:
+		return sum(
+			os.path.getsize(os.path.join(dirpath, filename))
+			for dirpath, _, filenames in os.walk(directory)
+			for filename in filenames
+		)
+
+	# Imports
+	try:
+		import zarr  # pyright: ignore[reportMissingTypeStubs]
+	except ImportError as e:
+		raise ImportError("zarr is required for array_to_disk function. Please install it via 'pip install zarr'.") from e
+
+	# If data is already a zarr.Array and more_data is present, just append and return
+	if isinstance(data, zarr.Array) and more_data is not None:
+		original_size: int = data.shape[0]
+		new_shape: tuple[int, ...] = (original_size + more_data.shape[0], *data.shape[1:])
+		data.resize(new_shape)
+		data[original_size:] = more_data[:]
+
+		# Delete more_data if specified, calculate size, and return
+		if delete_input:
+			del more_data
+		store_path: str = str(data.store.path if hasattr(data.store, 'path') else data.store) # type: ignore
+		return data, store_path, dir_size(store_path)
+
+	# Create a temporary directory to store the zarr array (with compression (auto-chunking for optimal performance))
+	temp_dir: str = tempfile.mkdtemp()
+	zarr_array: zarr.Array = zarr.open_array(temp_dir, mode="w", shape=data.shape, dtype=data.dtype, chunks=True) # pyright: ignore[reportUnknownMemberType]
+	zarr_array[:] = data[:]
+
+	# If additional data is provided, resize and append
+	if more_data is not None:
+		original_size = data.shape[0]
+		new_shape = (original_size + more_data.shape[0], *data.shape[1:])
+		zarr_array.resize(new_shape)
+		zarr_array[original_size:] = more_data[:]
+
+	# Delete the original data from memory if specified
+	if delete_input:
+		del data
+		if more_data is not None:
+			del more_data
+
+	# Register a cleanup function to delete the zarr directory at exit
+	atexit.register(lambda: shutil.rmtree(temp_dir, ignore_errors=True))
+
+	# Return all
+	return zarr_array, temp_dir, dir_size(temp_dir)
+
+if __name__ == "__main__":
+
+	# Example usage of array_to_disk (now using zarr)
+	print("\nZarr Example:")
+	data = np.random.rand(1000, 1000)
+	zarr_array, dir_path, total_size = array_to_disk(data, delete_input=True)
+	print(f"Zarr array shape: {zarr_array.shape}, directory: {dir_path}, size: {total_size:,} bytes")
+	print(f"Compression ratio: {(data.nbytes / total_size):.2f}x")
+
+	# Make it longer (1000x1000 -> 1500x1000)
+	data2 = np.random.rand(500, 1000)
+	longer_array, dir_path, total_size = array_to_disk(zarr_array, more_data=data2)
+	print(f"\nLonger zarr array shape: {longer_array.shape}, directory: {dir_path}, size: {total_size:,} bytes")
+	print(f"Compression ratio: {(1500 * 1000 * 8 / total_size):.2f}x")
+

stouputils/continuous_delivery/__init__.py

@@ -0,0 +1,27 @@
+""" Continuous delivery and deployment utilities.
+
+This module provides tools for automating software delivery and deployment:
+
+Key Features:
+- GitHub release management and uploads
+- PyPI package publishing utilities
+- pyproject.toml file management
+- Common CD/CI utilities
+
+Components:
+- cd_utils: Common utilities for continuous delivery
+- github: GitHub-specific utilities (upload_to_github)
+- pypi: PyPI publishing tools (pypi_full_routine)
+- pyproject: pyproject.toml file management
+- stubs: Stub file generation using pyright (stubs_full_routine)
+
+"""
+# ruff: noqa: F403
+
+# Imports
+from .cd_utils import *
+from .github import *
+from .pypi import *
+from .pyproject import *
+from .stubs import *
+

stouputils/continuous_delivery/cd_utils.py

@@ -0,0 +1,243 @@
+""" This module contains utilities for continuous delivery, such as loading credentials from a file.
+It is mainly used by the `stouputils.continuous_delivery.github` module.
+"""
+
+# Imports
+import os
+from typing import TYPE_CHECKING, Any
+
+from ..decorators import handle_error
+from ..io import clean_path, json_load
+from ..print import warning
+
+if TYPE_CHECKING:
+	import requests
+
+
+# Load credentials from file
+@handle_error()
+def load_credentials(credentials_path: str) -> dict[str, Any]:
+	""" Load credentials from a JSON or YAML file into a dictionary.
+
+	Loads credentials from either a JSON or YAML file and returns them as a dictionary.
+	The file must contain the required credentials in the appropriate format.
+
+	Args:
+		credentials_path (str): Path to the credentials file (.json or .yml)
+	Returns:
+		dict[str, Any]: Dictionary containing the credentials
+
+	Example JSON format:
+
+	.. code-block:: json
+
+		{
+			"github": {
+				"username": "Stoupy51",
+				"api_key": "ghp_XXXXXXXXXXXXXXXXXXXXXXXXXX"
+			}
+		}
+
+	Example YAML format:
+
+	.. code-block:: yaml
+
+		github:
+			username: "Stoupy51"
+			api_key: "ghp_XXXXXXXXXXXXXXXXXXXXXXXXXX"
+	"""
+	# Get the absolute path of the credentials file
+	warning(
+		"Be cautious when loading credentials from external sources like this, "
+		"as they might contain malicious code that could compromise your credentials without your knowledge"
+	)
+	credentials_path = clean_path(credentials_path)
+
+	# Check if the file exists
+	if not os.path.exists(credentials_path):
+		raise FileNotFoundError(f"Credentials file not found at '{credentials_path}'")
+
+	# Load the file if it's a JSON file
+	if credentials_path.endswith(".json"):
+		return json_load(credentials_path)
+
+	# Else, load the file if it's a YAML file
+	elif credentials_path.endswith((".yml", ".yaml")):
+		from msgspec import yaml
+		with open(credentials_path) as f:
+			return yaml.decode(f.read())
+
+	# Else, raise an error
+	else:
+		raise ValueError("Credentials file must be .json or .yml format")
+
+# Handle a response
+def handle_response(response: "requests.Response", error_message: str) -> None:
+	""" Handle a response from the API by raising an error if the response is not successful (status code not in 200-299).
+
+	Args:
+		response		(requests.Response): The response from the API
+		error_message	(str): The error message to raise if the response is not successful
+	"""
+	if response.status_code < 200 or response.status_code >= 300:
+		import requests
+		try:
+			raise ValueError(f"{error_message}, response code {response.status_code} with response {response.json()}")
+		except requests.exceptions.JSONDecodeError as e:
+			raise ValueError(f"{error_message}, response code {response.status_code} with response {response.text}") from e
+
+# Clean a version string
+def clean_version(version: str, keep: str = "") -> str:
+	""" Clean a version string
+
+	Args:
+		version	(str): The version string to clean
+		keep	(str): The characters to keep in the version string
+	Returns:
+		str: The cleaned version string
+
+	>>> clean_version("v1.e0.zfezf0.1.2.3zefz")
+	'1.0.0.1.2.3'
+	>>> clean_version("v1.e0.zfezf0.1.2.3zefz", keep="v")
+	'v1.0.0.1.2.3'
+	>>> clean_version("v1.2.3b", keep="ab")
+	'1.2.3b'
+	"""
+	return "".join(c for c in version if c in "0123456789." + keep)
+
+# Convert a version string to a float
+def version_to_float(version: str, error: bool = True) -> Any:
+	""" Converts a version string into a float for comparison purposes.
+	The version string is expected to follow the format of major.minor.patch.something_else....,
+	where each part is separated by a dot and can be extended indefinitely.
+	Supports pre-release suffixes with numbers: devN/dN (dev), aN (alpha), bN (beta), rcN/cN (release candidate).
+	Ordering: 1.0.0 > 1.0.0rc2 > 1.0.0rc1 > 1.0.0b2 > 1.0.0b1 > 1.0.0a2 > 1.0.0a1 > 1.0.0dev1
+
+	Args:
+		version (str): The version string to convert. (e.g. "v1.0.0.1.2.3", "v2.0.0b2", "v1.0.0rc1")
+		error (bool): Return None on error instead of raising an exception
+	Returns:
+		float: The float representation of the version. (e.g. 0)
+
+	>>> version_to_float("v1.0.0")
+	1.0
+	>>> version_to_float("v1.0.0.1")
+	1.000000001
+	>>> version_to_float("v2.3.7")
+	2.003007
+	>>> version_to_float("v1.0.0.1.2.3")
+	1.0000000010020031
+	>>> version_to_float("v2.0") > version_to_float("v1.0.0.1")
+	True
+	>>> version_to_float("v2.0.0") > version_to_float("v2.0.0rc") > version_to_float("v2.0.0b") > version_to_float("v2.0.0a") > version_to_float("v2.0.0dev")
+	True
+	>>> version_to_float("v1.0.0b") > version_to_float("v1.0.0a")
+	True
+	>>> version_to_float("v1.0.0") > version_to_float("v1.0.0b")
+	True
+	>>> version_to_float("v3.0.0a") > version_to_float("v2.9.9")
+	True
+	>>> version_to_float("v1.2.3b") < version_to_float("v1.2.3")
+	True
+	>>> version_to_float("1.0.0") == version_to_float("v1.0.0")
+	True
+	>>> version_to_float("2.0.0.0.0.0.1b") > version_to_float("2.0.0.0.0.0.1a")
+	True
+	>>> version_to_float("2.0.0.0.0.0.1") > version_to_float("2.0.0.0.0.0.1b")
+	True
+	>>> version_to_float("v1.0.0rc") == version_to_float("v1.0.0c")
+	True
+	>>> version_to_float("v1.0.0c") > version_to_float("v1.0.0b")
+	True
+	>>> version_to_float("v1.0.0d") < version_to_float("v1.0.0a")
+	True
+	>>> version_to_float("v1.0.0dev") < version_to_float("v1.0.0a")
+	True
+	>>> version_to_float("v1.0.0dev") == version_to_float("v1.0.0d")
+	True
+	>>> version_to_float("v1.0.0rc2") > version_to_float("v1.0.0rc1")
+	True
+	>>> version_to_float("v1.0.0b2") > version_to_float("v1.0.0b1")
+	True
+	>>> version_to_float("v1.0.0a2") > version_to_float("v1.0.0a1")
+	True
+	>>> version_to_float("v1.0.0dev2") > version_to_float("v1.0.0dev1")
+	True
+	>>> version_to_float("v1.0.0") > version_to_float("v1.0.0rc2") > version_to_float("v1.0.0rc1")
+	True
+	>>> version_to_float("v1.0.0rc1") > version_to_float("v1.0.0b2")
+	True
+	>>> version_to_float("v1.0.0b1") > version_to_float("v1.0.0a2")
+	True
+	>>> version_to_float("v1.0.0a1") > version_to_float("v1.0.0dev2")
+	True
+	>>> versions = ["v1.0.0", "v1.0.0rc2", "v1.0.0rc1", "v1.0.0b2", "v1.0.0b1", "v1.0.0a2", "v1.0.0a1", "v1.0.0dev2", "v1.0.0dev1"]
+	>>> sorted_versions = sorted(versions, key=version_to_float, reverse=True)
+	>>> sorted_versions == versions
+	True
+	"""
+	try:
+		# Check for pre-release suffixes and calculate suffix modifier
+		# Suffixes are ordered from longest to shortest to avoid partial matches
+		suffix_modifiers: dict[str, int] = {
+			"dev": 4,  # dev is lowest
+			"d": 4,    # d (dev) is lowest
+			"a": 3,    # alpha
+			"b": 2,    # beta
+			"rc": 1,   # rc is highest pre-release
+			"c": 1,    # c (release candidate)
+		}
+		suffix_type: int = 0  # 0 = no suffix, 1-4 = rc/c, b, a, dev/d
+		suffix_number: int = 0
+
+		# Check for suffixes with optional numbers
+		for suffix, modifier in suffix_modifiers.items():
+			if suffix in version:
+				# Find the suffix position
+				suffix_pos: int = version.rfind(suffix)
+				after_suffix: str = version[suffix_pos + len(suffix):]
+
+				# Check if there's a number after the suffix
+				if after_suffix.isdigit():
+					suffix_number = int(after_suffix)
+					version = version[:suffix_pos]
+				elif after_suffix == "":
+					# Suffix at the end without number
+					version = version[:suffix_pos]
+				else:
+					# Not a valid suffix match, continue searching
+					continue
+
+				# Found a valid suffix, set the type and break
+				suffix_type = modifier
+				break
+
+		# Clean the version string by keeping only the numbers and dots
+		version = clean_version(version)
+
+		# Split the version string into parts
+		version_parts: list[str] = version.split(".")
+		total: float = 0.0
+		multiplier: float = 1.0
+
+		# Iterate over the parts and add lesser and lesser weight to each part
+		for part in version_parts:
+			total += int(part) * multiplier
+			multiplier /= 1_000
+
+		# Apply pre-release modifier
+		# Pre-releases are represented as negative offsets from the base version
+		# Lower suffix_type = closer to release (rc=1 is closest, dev=4 is furthest)
+		# Higher suffix_number = closer to release within the same suffix type
+		# Formula: base_version - (suffix_type * 1000 - suffix_number) * 1e-9
+		# This ensures: 1.0.0 > 1.0.0rc2 > 1.0.0rc1 > 1.0.0b2 > 1.0.0a2 > 1.0.0dev2
+		if suffix_type > 0:
+			total -= (suffix_type * 1000 - suffix_number) * 1e-9
+
+		return total
+	except Exception as e:
+		if error:
+			raise ValueError(f"Invalid version string: '{version}'") from e
+		else:
+			return None # type: ignore
+