stouputils 1.3.16__tar.gz → 1.3.18__tar.gz

{stouputils-1.3.16 → stouputils-1.3.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stouputils
-Version: 1.3.16
+Version: 1.3.18
 Summary: Stouputils is a collection of utility modules designed to simplify and enhance the development process. It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers, and many more.
 Project-URL: Homepage, https://github.com/Stoupy51/stouputils
 Project-URL: Issues, https://github.com/Stoupy51/stouputils/issues

{stouputils-1.3.16 → stouputils-1.3.18}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "stouputils"
-version = "1.3.16"
+version = "1.3.18"
 description = "Stouputils is a collection of utility modules designed to simplify and enhance the development process. It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers, and many more."
 readme = "README.md"
 requires-python = ">=3.10"

{stouputils-1.3.16 → stouputils-1.3.18}/stouputils/backup.py

@@ -20,8 +20,12 @@ import zipfile
 # Local imports
 from .decorators import handle_error, measure_time
 from .io import clean_path
+from .parallel import colored_for_loop
 from .print import info, progress, warning
 
+# Constants
+CHUNK_SIZE = 1048576  # 1MB chunks for I/O operations
+
 
 # Function to compute the SHA-256 hash of a file
 def get_file_hash(file_path: str) -> str | None:
@@ -35,7 +39,11 @@ def get_file_hash(file_path: str) -> str | None:
 	try:
 		sha256_hash = hashlib.sha256()
 		with open(file_path, "rb") as f:
-			for chunk in iter(lambda: f.read(4096), b""):
+			# Use larger chunks for better I/O performance
+			while True:
+				chunk = f.read(CHUNK_SIZE)
+				if not chunk:
+					break
 				sha256_hash.update(chunk)
 		return sha256_hash.hexdigest()
 	except Exception as e:
@@ -175,10 +183,13 @@ def create_delta_backup(source_path: str, destination_folder: str, exclude_patte
 							zip_info.compress_type = zipfile.ZIP_DEFLATED
 							zip_info.comment = file_hash.encode()  # Store hash in comment
 
-							# Read and write file in chunks
+							# Read and write file in chunks with larger buffer
 							with open(full_path, "rb") as f:
 								with zipf.open(zip_info, "w", force_zip64=True) as zf:
-									for chunk in iter(lambda: f.read(4096), b""):
+									while True:
+										chunk = f.read(CHUNK_SIZE)
+										if not chunk:
+											break
 										zf.write(chunk)
 							has_changes = True
 						except Exception as e:
@@ -199,7 +210,10 @@ def create_delta_backup(source_path: str, destination_folder: str, exclude_patte
 
 					with open(source_path, "rb") as f:
 						with zipf.open(zip_info, "w", force_zip64=True) as zf:
-							for chunk in iter(lambda: f.read(4096), b""):
+							while True:
+								chunk = f.read(CHUNK_SIZE)
+								if not chunk:
+									break
 								zf.write(chunk)
 					has_changes = True
 				except Exception as e:
@@ -242,34 +256,69 @@ def consolidate_backups(zip_path: str, destination_zip: str) -> None:
 
 	# Get all previous backups up to the specified one
 	previous_backups: dict[str, dict[str, str]] = get_all_previous_backups(zip_folder, all_before=zip_path)
+	backup_paths: list[str] = list(previous_backups.keys())
 
+	# First pass: collect all deleted files and build file registry
 	deleted_files: set[str] = set()
-	final_files: set[str] = set()
+	file_registry: dict[str, tuple[str, zipfile.ZipInfo]] = {}  # filename -> (backup_path, zipinfo)
 
-	# Create destination ZIP file
-	with zipfile.ZipFile(destination_zip, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as zipf_out:
-		# Process each backup, tracking deleted files and consolidating files
-		for backup_path in previous_backups:
+	# Process backups in reverse order (newest first) to prioritize latest versions
+	for backup_path in reversed(backup_paths):
+		try:
 			with zipfile.ZipFile(backup_path, "r") as zipf_in:
+
+				# Get namelist once for efficiency
+				namelist: list[str] = zipf_in.namelist()
+
 				# Process deleted files
-				if "__deleted_files__.txt" in zipf_in.namelist():
+				if "__deleted_files__.txt" in namelist:
 					backup_deleted_files: list[str] = zipf_in.read("__deleted_files__.txt").decode().splitlines()
 					deleted_files.update(backup_deleted_files)
 
-				# Process files
+				# Process files - only add if not already in registry (newer versions take precedence)
 				for inf in zipf_in.infolist():
 					filename: str = inf.filename
-					if filename \
-						and filename != "__deleted_files__.txt" \
-						and filename not in final_files \
-						and filename not in deleted_files:
-						final_files.add(filename)
-
-						# Copy file in chunks
-						with zipf_in.open(inf, "r") as source:
-							with zipf_out.open(inf, "w", force_zip64=True) as target:
-								for chunk in iter(lambda: source.read(4096), b""):
-									target.write(chunk)
+					if (filename
+						and filename != "__deleted_files__.txt"
+						and filename not in deleted_files
+						and filename not in file_registry):
+						file_registry[filename] = (backup_path, inf)
+		except Exception as e:
+			warning(f"Error processing backup {backup_path}: {e}")
+			continue
+
+	# Second pass: copy files efficiently, keeping ZIP files open longer
+	open_zips: dict[str, zipfile.ZipFile] = {}
+
+	try:
+		with zipfile.ZipFile(destination_zip, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as zipf_out:
+			for filename, (backup_path, inf) in colored_for_loop(file_registry.items(), desc="Making consolidated backup"):
+				try:
+					# Open ZIP file if not already open
+					if backup_path not in open_zips:
+						open_zips[backup_path] = zipfile.ZipFile(backup_path, "r")
+
+					zipf_in = open_zips[backup_path]
+
+					# Copy file with larger chunks for better performance
+					with zipf_in.open(inf, "r") as source:
+						with zipf_out.open(inf, "w", force_zip64=True) as target:
+							# Use larger chunk size for better I/O performance
+							while True:
+								chunk = source.read(CHUNK_SIZE)
+								if not chunk:
+									break
+								target.write(chunk)
+				except Exception as e:
+					warning(f"Error copying file {filename} from {backup_path}: {e}")
+					continue
+	finally:
+		# Clean up open ZIP files
+		for zipf in open_zips.values():
+			try:
+				zipf.close()
+			except Exception:
+				pass
 
 	info(f"Consolidated backup created: {destination_zip}")
 

{stouputils-1.3.16 → stouputils-1.3.18}/stouputils/parallel.py

@@ -11,9 +11,9 @@ I highly encourage you to read the function docstrings to understand when to use
 """
 
 # Imports
-import time
 import multiprocessing as mp
-from collections.abc import Callable, Iterator
+import time
+from collections.abc import Callable, Iterable, Iterator
 from concurrent.futures import ThreadPoolExecutor
 from multiprocessing import Pool, cpu_count
 from typing import Any, TypeVar
@@ -179,7 +179,7 @@ def multiprocessing(
 				old_method: str | None = mp.get_start_method(allow_none=True)
 				new_method: str = "spawn" if old_method in (None, "fork") else "fork"
 				mp.set_start_method(new_method, force=True)
-				
+
 				try:
 					return process()
 				finally:
@@ -278,7 +278,7 @@ def multithreading(
 
 
 def colored_for_loop(
-	iterable: list[T],
+	iterable: Iterable[T],
 	desc: str = "Processing",
 	color: str = MAGENTA,
 	bar_format: str = BAR_FORMAT,
@@ -288,7 +288,7 @@ def colored_for_loop(
 	""" Function to iterate over a list with a colored TQDM progress bar like the other functions in this module.
 
 	Args:
-		iterable	(list):				List to iterate over
+		iterable	(Iterable):			List to iterate over
 		desc		(str):				Description of the function execution displayed in the progress bar
 		color		(str):				Color of the progress bar (Defaults to MAGENTA)
 		bar_format	(str):				Format of the progress bar (Defaults to BAR_FORMAT)