stouputils 1.14.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. stouputils/__init__.py +40 -0
  2. stouputils/__main__.py +86 -0
  3. stouputils/_deprecated.py +37 -0
  4. stouputils/all_doctests.py +160 -0
  5. stouputils/applications/__init__.py +22 -0
  6. stouputils/applications/automatic_docs.py +634 -0
  7. stouputils/applications/upscaler/__init__.py +39 -0
  8. stouputils/applications/upscaler/config.py +128 -0
  9. stouputils/applications/upscaler/image.py +247 -0
  10. stouputils/applications/upscaler/video.py +287 -0
  11. stouputils/archive.py +344 -0
  12. stouputils/backup.py +488 -0
  13. stouputils/collections.py +244 -0
  14. stouputils/continuous_delivery/__init__.py +27 -0
  15. stouputils/continuous_delivery/cd_utils.py +243 -0
  16. stouputils/continuous_delivery/github.py +522 -0
  17. stouputils/continuous_delivery/pypi.py +130 -0
  18. stouputils/continuous_delivery/pyproject.py +147 -0
  19. stouputils/continuous_delivery/stubs.py +86 -0
  20. stouputils/ctx.py +408 -0
  21. stouputils/data_science/config/get.py +51 -0
  22. stouputils/data_science/config/set.py +125 -0
  23. stouputils/data_science/data_processing/image/__init__.py +66 -0
  24. stouputils/data_science/data_processing/image/auto_contrast.py +79 -0
  25. stouputils/data_science/data_processing/image/axis_flip.py +58 -0
  26. stouputils/data_science/data_processing/image/bias_field_correction.py +74 -0
  27. stouputils/data_science/data_processing/image/binary_threshold.py +73 -0
  28. stouputils/data_science/data_processing/image/blur.py +59 -0
  29. stouputils/data_science/data_processing/image/brightness.py +54 -0
  30. stouputils/data_science/data_processing/image/canny.py +110 -0
  31. stouputils/data_science/data_processing/image/clahe.py +92 -0
  32. stouputils/data_science/data_processing/image/common.py +30 -0
  33. stouputils/data_science/data_processing/image/contrast.py +53 -0
  34. stouputils/data_science/data_processing/image/curvature_flow_filter.py +74 -0
  35. stouputils/data_science/data_processing/image/denoise.py +378 -0
  36. stouputils/data_science/data_processing/image/histogram_equalization.py +123 -0
  37. stouputils/data_science/data_processing/image/invert.py +64 -0
  38. stouputils/data_science/data_processing/image/laplacian.py +60 -0
  39. stouputils/data_science/data_processing/image/median_blur.py +52 -0
  40. stouputils/data_science/data_processing/image/noise.py +59 -0
  41. stouputils/data_science/data_processing/image/normalize.py +65 -0
  42. stouputils/data_science/data_processing/image/random_erase.py +66 -0
  43. stouputils/data_science/data_processing/image/resize.py +69 -0
  44. stouputils/data_science/data_processing/image/rotation.py +80 -0
  45. stouputils/data_science/data_processing/image/salt_pepper.py +68 -0
  46. stouputils/data_science/data_processing/image/sharpening.py +55 -0
  47. stouputils/data_science/data_processing/image/shearing.py +64 -0
  48. stouputils/data_science/data_processing/image/threshold.py +64 -0
  49. stouputils/data_science/data_processing/image/translation.py +71 -0
  50. stouputils/data_science/data_processing/image/zoom.py +83 -0
  51. stouputils/data_science/data_processing/image_augmentation.py +118 -0
  52. stouputils/data_science/data_processing/image_preprocess.py +183 -0
  53. stouputils/data_science/data_processing/prosthesis_detection.py +359 -0
  54. stouputils/data_science/data_processing/technique.py +481 -0
  55. stouputils/data_science/dataset/__init__.py +45 -0
  56. stouputils/data_science/dataset/dataset.py +292 -0
  57. stouputils/data_science/dataset/dataset_loader.py +135 -0
  58. stouputils/data_science/dataset/grouping_strategy.py +296 -0
  59. stouputils/data_science/dataset/image_loader.py +100 -0
  60. stouputils/data_science/dataset/xy_tuple.py +696 -0
  61. stouputils/data_science/metric_dictionnary.py +106 -0
  62. stouputils/data_science/metric_utils.py +847 -0
  63. stouputils/data_science/mlflow_utils.py +206 -0
  64. stouputils/data_science/models/abstract_model.py +149 -0
  65. stouputils/data_science/models/all.py +85 -0
  66. stouputils/data_science/models/base_keras.py +765 -0
  67. stouputils/data_science/models/keras/all.py +38 -0
  68. stouputils/data_science/models/keras/convnext.py +62 -0
  69. stouputils/data_science/models/keras/densenet.py +50 -0
  70. stouputils/data_science/models/keras/efficientnet.py +60 -0
  71. stouputils/data_science/models/keras/mobilenet.py +56 -0
  72. stouputils/data_science/models/keras/resnet.py +52 -0
  73. stouputils/data_science/models/keras/squeezenet.py +233 -0
  74. stouputils/data_science/models/keras/vgg.py +42 -0
  75. stouputils/data_science/models/keras/xception.py +38 -0
  76. stouputils/data_science/models/keras_utils/callbacks/__init__.py +20 -0
  77. stouputils/data_science/models/keras_utils/callbacks/colored_progress_bar.py +219 -0
  78. stouputils/data_science/models/keras_utils/callbacks/learning_rate_finder.py +148 -0
  79. stouputils/data_science/models/keras_utils/callbacks/model_checkpoint_v2.py +31 -0
  80. stouputils/data_science/models/keras_utils/callbacks/progressive_unfreezing.py +249 -0
  81. stouputils/data_science/models/keras_utils/callbacks/warmup_scheduler.py +66 -0
  82. stouputils/data_science/models/keras_utils/losses/__init__.py +12 -0
  83. stouputils/data_science/models/keras_utils/losses/next_generation_loss.py +56 -0
  84. stouputils/data_science/models/keras_utils/visualizations.py +416 -0
  85. stouputils/data_science/models/model_interface.py +939 -0
  86. stouputils/data_science/models/sandbox.py +116 -0
  87. stouputils/data_science/range_tuple.py +234 -0
  88. stouputils/data_science/scripts/augment_dataset.py +77 -0
  89. stouputils/data_science/scripts/exhaustive_process.py +133 -0
  90. stouputils/data_science/scripts/preprocess_dataset.py +70 -0
  91. stouputils/data_science/scripts/routine.py +168 -0
  92. stouputils/data_science/utils.py +285 -0
  93. stouputils/decorators.py +605 -0
  94. stouputils/image.py +441 -0
  95. stouputils/installer/__init__.py +18 -0
  96. stouputils/installer/common.py +67 -0
  97. stouputils/installer/downloader.py +101 -0
  98. stouputils/installer/linux.py +144 -0
  99. stouputils/installer/main.py +223 -0
  100. stouputils/installer/windows.py +136 -0
  101. stouputils/io.py +486 -0
  102. stouputils/parallel.py +483 -0
  103. stouputils/print.py +482 -0
  104. stouputils/py.typed +1 -0
  105. stouputils/stouputils/__init__.pyi +15 -0
  106. stouputils/stouputils/_deprecated.pyi +12 -0
  107. stouputils/stouputils/all_doctests.pyi +46 -0
  108. stouputils/stouputils/applications/__init__.pyi +2 -0
  109. stouputils/stouputils/applications/automatic_docs.pyi +106 -0
  110. stouputils/stouputils/applications/upscaler/__init__.pyi +3 -0
  111. stouputils/stouputils/applications/upscaler/config.pyi +18 -0
  112. stouputils/stouputils/applications/upscaler/image.pyi +109 -0
  113. stouputils/stouputils/applications/upscaler/video.pyi +60 -0
  114. stouputils/stouputils/archive.pyi +67 -0
  115. stouputils/stouputils/backup.pyi +109 -0
  116. stouputils/stouputils/collections.pyi +86 -0
  117. stouputils/stouputils/continuous_delivery/__init__.pyi +5 -0
  118. stouputils/stouputils/continuous_delivery/cd_utils.pyi +129 -0
  119. stouputils/stouputils/continuous_delivery/github.pyi +162 -0
  120. stouputils/stouputils/continuous_delivery/pypi.pyi +53 -0
  121. stouputils/stouputils/continuous_delivery/pyproject.pyi +67 -0
  122. stouputils/stouputils/continuous_delivery/stubs.pyi +39 -0
  123. stouputils/stouputils/ctx.pyi +211 -0
  124. stouputils/stouputils/decorators.pyi +252 -0
  125. stouputils/stouputils/image.pyi +172 -0
  126. stouputils/stouputils/installer/__init__.pyi +5 -0
  127. stouputils/stouputils/installer/common.pyi +39 -0
  128. stouputils/stouputils/installer/downloader.pyi +24 -0
  129. stouputils/stouputils/installer/linux.pyi +39 -0
  130. stouputils/stouputils/installer/main.pyi +57 -0
  131. stouputils/stouputils/installer/windows.pyi +31 -0
  132. stouputils/stouputils/io.pyi +213 -0
  133. stouputils/stouputils/parallel.pyi +216 -0
  134. stouputils/stouputils/print.pyi +136 -0
  135. stouputils/stouputils/version_pkg.pyi +15 -0
  136. stouputils/version_pkg.py +189 -0
  137. stouputils-1.14.0.dist-info/METADATA +178 -0
  138. stouputils-1.14.0.dist-info/RECORD +140 -0
  139. stouputils-1.14.0.dist-info/WHEEL +4 -0
  140. stouputils-1.14.0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,18 @@
1
+ WAIFU2X_NCNN_VULKAN_RELEASES: dict[str, str]
2
+ FFMPEG_RELEASES: dict[str, str]
3
+ YOUTUBE_BITRATE_RECOMMENDATIONS: dict[str, dict[str, dict[int, int]]]
4
+
5
+ class Config:
6
+ """ Configuration class for the upscaler. """
7
+ JPG_QUALITY: int
8
+ VIDEO_FINAL_BITRATE: int
9
+ FFMPEG_EXECUTABLE: str
10
+ FFMPEG_ARGS: tuple[str, ...]
11
+ FFPROBE_EXECUTABLE: str
12
+ FFMPEG_CHECK_HELP_TEXT: str
13
+ UPSCALER_EXECUTABLE: str
14
+ UPSCALER_ARGS: tuple[str, ...]
15
+ UPSCALER_EXECUTABLE_HELP_TEXT: str
16
+ SLIGHTLY_FASTER_MODE: bool
17
+ upscaler_executable_checked: bool
18
+ ffmpeg_executable_checked: bool
@@ -0,0 +1,109 @@
1
+ from ...installer import check_executable as check_executable
2
+ from ...io import clean_path as clean_path
3
+ from ...parallel import multithreading as multithreading
4
+ from ...print import colored_for_loop as colored_for_loop, debug as debug, info as info
5
+ from .config import Config as Config, WAIFU2X_NCNN_VULKAN_RELEASES as WAIFU2X_NCNN_VULKAN_RELEASES
6
+ from tempfile import TemporaryDirectory
7
+
8
+ def convert_frame(frame_path: str, delete_png: bool = True) -> None:
9
+ ''' Convert a PNG frame to JPG format to take less space.
10
+
11
+ \tArgs:
12
+ \t\tframe_path (str): Path to the PNG frame to convert.
13
+ \t\tdelete_png (bool): Whether to delete the original PNG file after conversion.
14
+
15
+ \tReturns:
16
+ \t\tNone: This function doesn\'t return anything.
17
+
18
+ \tExample:
19
+ \t\t.. code-block:: python
20
+
21
+ \t\t\t> convert_frame("input.png", delete_png=True)
22
+ \t\t\t> # input.png will be converted to input.jpg and the original file will be deleted
23
+
24
+ \t\t\t> convert_frame("input.png", delete_png=False)
25
+ \t\t\t> # input.png will be converted to input.jpg and the original file will be kept
26
+ \t'''
27
+ def get_all_files(folder: str, suffix: str | tuple[str, ...] = '') -> list[str]:
28
+ ''' Get all files paths in a folder, with a specific suffix if provided.
29
+
30
+ \tArgs:
31
+ \t\tfolder (str): Path to the folder containing the files.
32
+ \t\tsuffix (str | tuple[str, ...]): Suffix of the files to get (e.g. ".png", ".jpg", etc.).
33
+
34
+ \tReturns:
35
+ \t\tlist[str]: List of all files paths in the folder.
36
+
37
+ \tExample:
38
+ \t\t>>> files: list[str] = get_all_files("some_folder", ".png")
39
+ \t\t>>> len(files)
40
+ \t\t0
41
+ \t'''
42
+ def create_temp_dir_for_not_upscaled(input_path: str, output_path: str) -> TemporaryDirectory[str] | None:
43
+ """ Creates a temporary directory containing only images that haven't been upscaled yet.
44
+
45
+ Args:
46
+ input_path (str): Path to the folder containing input images.
47
+ output_path (str): Path to the folder where upscaled images are saved.
48
+
49
+ Returns:
50
+ TemporaryDirectory[str] | None: A temporary directory object if there are images to process,
51
+ None if all images are already upscaled.
52
+ """
53
+ def check_upscaler_executable() -> None: ...
54
+ def upscale(input_path: str, output_path: str, upscale_ratio: int) -> None:
55
+ ''' Upscale an input image (or a directory of images) with the upscaler executable.
56
+
57
+ \tArgs:
58
+ \t\tinput_path (str): Path to the image to upscale (or a directory).
59
+ \t\toutput_path (str): Path to the output image (or a directory).
60
+ \t\tupscale_ratio (int): Upscaling ratio.
61
+
62
+ \tExample:
63
+ \t\t.. code-block:: python
64
+
65
+ \t\t\t> upscale("folder", "folder", 2)
66
+ \t\t\tTraceback (most recent call last):
67
+ \t\t\t\t...
68
+ \t\t\tAssertionError: Input and output paths cannot be the same, got \'folder\'
69
+
70
+ \t\t\t> upscale("stouputils", "stouputils/output.jpg", 2)
71
+ \t\t\tTraceback (most recent call last):
72
+ \t\t\t\t...
73
+ \t\t\tAssertionError: If input is a directory, output must be a directory too, got \'stouputils/output.jpg\'
74
+
75
+
76
+ \t\t\t> upscale("input.jpg", "output.jpg", 2)
77
+ \t\t\t> # The input.jpg will be upscaled to output.jpg with a ratio of 2
78
+
79
+ \t\t\t> upscale("input_folder", "output_folder", 2)
80
+ \t\t\t> # The input_folder will be upscaled to output_folder with a ratio of 2
81
+ \t'''
82
+ def upscale_images(images: list[str], output_folder: str, upscale_ratio: int, desc: str = 'Upscaling images') -> None:
83
+ """ Upscale multiple images from a list.
84
+
85
+ \tArgs:
86
+ \t\timages (list[str]): List of paths to the images to upscale.
87
+ \t\toutput_folder (str): Path to the output folder where the upscaled images will be saved.
88
+ \t\tupscale_ratio (int): Upscaling ratio.
89
+ \t\tdesc (str): Description of the function execution displayed in the progress bar.
90
+ \t\t\tNo progress bar will be displayed if desc is empty.
91
+
92
+ \tReturns:
93
+ \t\tNone: This function doesn't return anything.
94
+ \t"""
95
+ def upscale_folder(input_folder: str, output_folder: str, upscale_ratio: int, slightly_faster_mode: bool = True, desc: str = 'Upscaling folder') -> None:
96
+ """ Upscale all images in a folder.
97
+
98
+ \tArgs:
99
+ \t\tinput_folder (str): Path to the input folder containing the images to upscale.
100
+ \t\toutput_folder (str): Path to the output folder where the upscaled images will be saved.
101
+ \t\tupscale_ratio (int): Upscaling ratio.
102
+ \t\tslightly_faster_mode (bool): Whether to use the slightly faster mode (no progress bar),
103
+ \t\t\tone call to the upscaler executable.
104
+ \t\tdesc (str): Description of the function execution displayed in the progress bar.
105
+ \t\t\tNo progress bar will be displayed if desc is empty.
106
+
107
+ \tReturns:
108
+ \t\tNone: This function doesn't return anything.
109
+ \t"""
@@ -0,0 +1,60 @@
1
+ from ...installer import check_executable as check_executable
2
+ from ...io import clean_path as clean_path
3
+ from ...parallel import multithreading as multithreading
4
+ from ...print import colored_for_loop as colored_for_loop, debug as debug, error as error, info as info, warning as warning
5
+ from .config import Config as Config, FFMPEG_RELEASES as FFMPEG_RELEASES, YOUTUBE_BITRATE_RECOMMENDATIONS as YOUTUBE_BITRATE_RECOMMENDATIONS
6
+ from .image import convert_frame as convert_frame, get_all_files as get_all_files, upscale_folder as upscale_folder
7
+ from typing import Literal
8
+
9
+ def get_recommended_bitrate(resolution: tuple[int, int], frame_rate: int = 60, upload_type: Literal['SDR', 'HDR'] = 'SDR') -> int:
10
+ ''' Get the recommended bitrate (in kbps) for the output video based on the video resolution.
11
+
12
+ \tArgs:
13
+ \t\tresolution (tuple[int, int]): Video resolution (width, height).
14
+ \t\tframe_rate (int): Frame rate of the video, default is 60.
15
+ \t\tupload_type (Literal["SDR","HDR"]): Upload type from which the recommendation is made, default is "SDR".
16
+
17
+ \tReturns:
18
+ \t\tint: The recommended bitrate for the output video (in kbps)
19
+
20
+ \tSource: https://support.google.com/youtube/answer/1722171?hl=en#zippy=%2Cbitrate
21
+
22
+ \tExamples:
23
+ \t\t>>> # Valid examples
24
+ \t\t>>> get_recommended_bitrate((3840, 2160), 60, "SDR")
25
+ \t\t68000
26
+ \t\t>>> get_recommended_bitrate((1920, 1080), 60, "HDR")
27
+ \t\t15000
28
+ \t\t>>> get_recommended_bitrate((1920, 1080), 60, "SDR")
29
+ \t\t12000
30
+ \t\t>>> get_recommended_bitrate((1920, 1080), 30, "SDR")
31
+ \t\t8000
32
+
33
+ \t\t>>> # Invalid examples
34
+ \t\t>>> get_recommended_bitrate((1920, 1080), 60, "Ratio")
35
+ \t\tTraceback (most recent call last):
36
+ \t\t\t...
37
+ \t\tAssertionError: Invalid upload type: \'Ratio\'
38
+ \t\t>>> get_recommended_bitrate("1920x1080", 60, "SDR")
39
+ \t\tTraceback (most recent call last):
40
+ \t\t\t...
41
+ \t\tAssertionError: Invalid resolution: 1920x1080, must be a tuple of two integers
42
+ \t\t>>> get_recommended_bitrate((1920, 1080), -10, "SDR")
43
+ \t\tTraceback (most recent call last):
44
+ \t\t\t...
45
+ \t\tAssertionError: Invalid frame rate: -10, must be a positive integer
46
+ \t'''
47
+ def check_ffmpeg_executable() -> None: ...
48
+ def upscale_video(video_file: str, input_folder: str, progress_folder: str, output_folder: str) -> None:
49
+ """ Handles a video file. """
50
+ def video_upscaler_cli(input_folder: str, progress_folder: str, output_folder: str) -> None:
51
+ """ Upscales videos from an input folder and saves them to an output folder.
52
+
53
+ \tUses intermediate folders for extracted and upscaled frames within the progress folder.
54
+ \t**Handles resuming partially processed videos.**
55
+
56
+ \tArgs:
57
+ \t\tinput_folder (str): Path to the folder containing input videos.
58
+ \t\tprogress_folder (str): Path to the folder for storing intermediate files (frames).
59
+ \t\toutput_folder (str): Path to the folder where upscaled videos will be saved.
60
+ \t"""
@@ -0,0 +1,67 @@
1
+ from .decorators import LogLevels as LogLevels, handle_error as handle_error
2
+ from .io import clean_path as clean_path, super_copy as super_copy
3
+ from .print import CYAN as CYAN, GREEN as GREEN, RESET as RESET, debug as debug, error as error, info as info
4
+
5
+ def repair_zip_file(file_path: str, destination: str) -> bool:
6
+ ''' Try to repair a corrupted zip file by ignoring some of the errors
7
+
8
+ \tThis function manually parses the ZIP file structure to extract files
9
+ \teven when the ZIP file is corrupted. It reads the central directory
10
+ \tentries and attempts to decompress each file individually.
11
+
12
+ \tArgs:
13
+ \t\tfile_path\t\t(str):\tPath of the zip file to repair
14
+ \t\tdestination\t\t(str):\tDestination of the new file
15
+ \tReturns:
16
+ \t\tbool: Always returns True unless any strong error
17
+
18
+ \tExamples:
19
+
20
+ \t.. code-block:: python
21
+
22
+ \t\t> repair_zip_file("/path/to/source.zip", "/path/to/destination.zip")
23
+ \t'''
24
+ def make_archive(source: str, destinations: list[str] | str | None = None, override_time: None | tuple[int, int, int, int, int, int] = None, create_dir: bool = False, ignore_patterns: str | None = None) -> bool:
25
+ ''' Create a zip archive from a source directory with consistent file timestamps.
26
+ \t(Meaning deterministic zip file each time)
27
+
28
+ \tCreates a zip archive from the source directory and copies it to one or more destinations.
29
+ \tThe archive will have consistent file timestamps across runs if override_time is specified.
30
+ \tUses maximum compression level (9) with ZIP_DEFLATED algorithm.
31
+
32
+ \tArgs:
33
+ \t\tsource\t\t\t\t(str):\t\t\t\t\t\tThe source folder to archive
34
+ \t\tdestinations\t\t(list[str]|str):\t\t\tThe destination folder(s) or file(s) to copy the archive to
35
+ \t\toverride_time\t\t(None | tuple[int, ...]):\tThe constant time to use for the archive
36
+ \t\t\t(e.g. (2024, 1, 1, 0, 0, 0) for 2024-01-01 00:00:00)
37
+ \t\tcreate_dir\t\t\t(bool):\t\t\t\t\t\tWhether to create the destination directory if it doesn\'t exist
38
+ \t\tignore_patterns\t\t(str | None):\t\t\t\tGlob pattern(s) to ignore files. Can be a single pattern or comma-separated patterns (e.g. "*.pyc" or "*.pyc,__pycache__,*.log")
39
+ \tReturns:
40
+ \t\tbool: Always returns True unless any strong error
41
+ \tExamples:
42
+
43
+ \t.. code-block:: python
44
+
45
+ \t\t> make_archive("/path/to/source", "/path/to/destination.zip")
46
+ \t\t> make_archive("/path/to/source", ["/path/to/destination.zip", "/path/to/destination2.zip"])
47
+ \t\t> make_archive("src", "hello_from_year_2085.zip", override_time=(2085,1,1,0,0,0))
48
+ \t\t> make_archive("src", "output.zip", ignore_patterns="*.pyc")
49
+ \t\t> make_archive("src", "output.zip", ignore_patterns="__pycache__")
50
+ \t\t> make_archive("src", "output.zip", ignore_patterns="*.pyc,__pycache__,*.log")
51
+ \t'''
52
+ def archive_cli() -> None:
53
+ ''' Main entry point for command line usage.
54
+
55
+ \tExamples:
56
+
57
+ \t.. code-block:: bash
58
+
59
+ \t\t# Repair a corrupted zip file
60
+ \t\tpython -m stouputils.archive repair /path/to/corrupted.zip /path/to/repaired.zip
61
+
62
+ \t\t# Create a zip archive
63
+ \t\tpython -m stouputils.archive make /path/to/source /path/to/destination.zip
64
+
65
+ \t\t# Create a zip archive with ignore patterns
66
+ \t\tpython -m stouputils.archive make /path/to/source /path/to/destination.zip --ignore "*.pyc,__pycache__"
67
+ \t'''
@@ -0,0 +1,109 @@
1
+ import zipfile
2
+ from .decorators import handle_error as handle_error, measure_time as measure_time
3
+ from .io import clean_path as clean_path
4
+ from .print import CYAN as CYAN, GREEN as GREEN, RESET as RESET, colored_for_loop as colored_for_loop, info as info, warning as warning
5
+
6
+ CHUNK_SIZE: int
7
+ LARGE_CHUNK_SIZE: int
8
+
9
+ def backup_cli() -> None:
10
+ ''' Main entry point for command line usage.
11
+
12
+ \tExamples:
13
+
14
+ \t.. code-block:: bash
15
+
16
+ \t\t# Create a delta backup, excluding libraries and cache folders
17
+ \t\tpython -m stouputils.backup delta /path/to/source /path/to/backups -x "libraries/*" "cache/*"
18
+
19
+ \t\t# Consolidate backups into a single file
20
+ \t\tpython -m stouputils.backup consolidate /path/to/backups/latest.zip /path/to/consolidated.zip
21
+
22
+ \t\t# Limit the number of delta backups to 5
23
+ \t\tpython -m stouputils.backup limit 5 /path/to/backups
24
+ \t'''
25
+ def create_delta_backup(source_path: str, destination_folder: str, exclude_patterns: list[str] | None = None) -> None:
26
+ ''' Creates a ZIP delta backup, saving only modified or new files while tracking deleted files.
27
+
28
+ \tArgs:
29
+ \t\tsource_path (str): Path to the source file or directory to back up
30
+ \t\tdestination_folder (str): Path to the folder where the backup will be saved
31
+ \t\texclude_patterns (list[str] | None): List of glob patterns to exclude from backup
32
+ \tExamples:
33
+
34
+ \t.. code-block:: python
35
+
36
+ \t\t> create_delta_backup("/path/to/source", "/path/to/backups", exclude_patterns=["libraries/*", "cache/*"])
37
+ \t\t[INFO HH:MM:SS] Creating ZIP backup
38
+ \t\t[INFO HH:MM:SS] Backup created: \'/path/to/backups/backup_2025_02_18-10_00_00.zip\'
39
+ \t'''
40
+ def consolidate_backups(zip_path: str, destination_zip: str) -> None:
41
+ ''' Consolidates the files from the given backup and all previous ones into a new ZIP file,
42
+ \tensuring that the most recent version of each file is kept and deleted files are not restored.
43
+
44
+ \tArgs:
45
+ \t\tzip_path (str): Path to the latest backup ZIP file (If endswith "/latest.zip" or "/", the latest backup will be used)
46
+ \t\tdestination_zip (str): Path to the destination ZIP file where the consolidated backup will be saved
47
+ \tExamples:
48
+
49
+ \t.. code-block:: python
50
+
51
+ \t\t> consolidate_backups("/path/to/backups/latest.zip", "/path/to/consolidated.zip")
52
+ \t\t[INFO HH:MM:SS] Consolidating backups
53
+ \t\t[INFO HH:MM:SS] Consolidated backup created: \'/path/to/consolidated.zip\'
54
+ \t'''
55
+ def limit_backups(max_backups: int, backup_folder: str, keep_oldest: bool = True) -> None:
56
+ ''' Limits the number of delta backups by consolidating the oldest ones.
57
+
58
+ \tIf the number of backups exceeds max_backups, the oldest backups are consolidated
59
+ \tinto a single backup file, then deleted, until the count is within the limit.
60
+
61
+ \tArgs:
62
+ \t\tmax_backups (int): Maximum number of delta backups to keep
63
+ \t\tbackup_folder (str): Path to the folder containing backups
64
+ \t\tkeep_oldest (bool): If True, never delete the oldest backup (default: True)
65
+ \tExamples:
66
+
67
+ \t.. code-block:: python
68
+
69
+ \t\t> limit_backups(5, "/path/to/backups")
70
+ \t\t[INFO HH:MM:SS] Limiting backups
71
+ \t\t[INFO HH:MM:SS] Consolidated 3 oldest backups into \'/path/to/backups/consolidated_YYYY_MM_DD-HH_MM_SS.zip\'
72
+ \t\t[INFO HH:MM:SS] Deleted 3 old backups
73
+ \t'''
74
+ def get_file_hash(file_path: str) -> str | None:
75
+ """ Computes the SHA-256 hash of a file.
76
+
77
+ \tArgs:
78
+ \t\tfile_path (str): Path to the file
79
+ \tReturns:
80
+ \t\tstr | None: SHA-256 hash as a hexadecimal string or None if an error occurs
81
+ \t"""
82
+ def extract_hash_from_zipinfo(zip_info: zipfile.ZipInfo) -> str | None:
83
+ """ Extracts the stored hash from a ZipInfo object's comment.
84
+
85
+ \tArgs:
86
+ \t\tzip_info (zipfile.ZipInfo): The ZipInfo object representing a file in the ZIP
87
+ \tReturns:
88
+ \t\tstr | None: The stored hash if available, otherwise None
89
+ \t"""
90
+ def get_all_previous_backups(backup_folder: str, all_before: str | None = None) -> dict[str, dict[str, str]]:
91
+ ''' Retrieves all previous backups in a folder and maps each backup to a dictionary of file paths and their hashes.
92
+
93
+ \tArgs:
94
+ \t\tbackup_folder (str): The folder containing previous backup zip files
95
+ \t\tall_before (str | None): Path to the latest backup ZIP file
96
+ \t\t\t(If endswith "/latest.zip" or "/", the latest backup will be used)
97
+ \tReturns:
98
+ \t\tdict[str, dict[str, str]]: Dictionary mapping backup file paths to dictionaries of {file_path: file_hash}
99
+ \t'''
100
+ def is_file_in_any_previous_backup(file_path: str, file_hash: str, previous_backups: dict[str, dict[str, str]]) -> bool:
101
+ """ Checks if a file with the same hash exists in any previous backup.
102
+
103
+ \tArgs:
104
+ \t\tfile_path (str): The relative path of the file
105
+ \t\tfile_hash (str): The SHA-256 hash of the file
106
+ \t\tprevious_backups (dict[str, dict[str, str]]): Dictionary mapping backup zip paths to their stored file hashes
107
+ \tReturns:
108
+ \t\tbool: True if the file exists unchanged in any previous backup, False otherwise
109
+ \t"""
@@ -0,0 +1,86 @@
1
+ import polars as pl
2
+ import zarr
3
+ from collections.abc import Iterable
4
+ from numpy.typing import NDArray as NDArray
5
+ from typing import Any, Literal, TypeVar
6
+
7
+ T = TypeVar('T')
8
+
9
+ def unique_list[T](list_to_clean: Iterable[T], method: Literal['id', 'hash', 'str'] = 'str') -> list[T]:
10
+ ''' Remove duplicates from the list while keeping the order using ids (default) or hash or str
11
+
12
+ \tArgs:
13
+ \t\tlist_to_clean\t(Iterable[T]):\t\t\t\t\tThe list to clean
14
+ \t\tmethod\t\t\t(Literal["id", "hash", "str"]):\tThe method to use to identify duplicates
15
+ \tReturns:
16
+ \t\tlist[T]: The cleaned list
17
+
18
+ \tExamples:
19
+ \t\t>>> unique_list([1, 2, 3, 2, 1], method="id")
20
+ \t\t[1, 2, 3]
21
+
22
+ \t\t>>> s1 = {1, 2, 3}
23
+ \t\t>>> s2 = {2, 3, 4}
24
+ \t\t>>> s3 = {1, 2, 3}
25
+ \t\t>>> unique_list([s1, s2, s1, s1, s3, s2, s3], method="id")
26
+ \t\t[{1, 2, 3}, {2, 3, 4}, {1, 2, 3}]
27
+
28
+ \t\t>>> s1 = {1, 2, 3}
29
+ \t\t>>> s2 = {2, 3, 4}
30
+ \t\t>>> s3 = {1, 2, 3}
31
+ \t\t>>> unique_list([s1, s2, s1, s1, s3, s2, s3], method="str")
32
+ \t\t[{1, 2, 3}, {2, 3, 4}]
33
+ \t'''
34
+ def sort_dict_keys[T](dictionary: dict[T, Any], order: list[T], reverse: bool = False) -> dict[T, Any]:
35
+ ''' Sort dictionary keys using a given order list (reverse optional)
36
+
37
+ \tArgs:
38
+ \t\tdictionary\t(dict[T, Any]):\tThe dictionary to sort
39
+ \t\torder\t\t(list[T]):\t\tThe order list
40
+ \t\treverse\t\t(bool):\t\t\tWhether to sort in reverse order (given to sorted function which behaves differently than order.reverse())
41
+ \tReturns:
42
+ \t\tdict[T, Any]: The sorted dictionary
43
+
44
+ \tExamples:
45
+ \t\t>>> sort_dict_keys({\'b\': 2, \'a\': 1, \'c\': 3}, order=["a", "b", "c"])
46
+ \t\t{\'a\': 1, \'b\': 2, \'c\': 3}
47
+
48
+ \t\t>>> sort_dict_keys({\'b\': 2, \'a\': 1, \'c\': 3}, order=["a", "b", "c"], reverse=True)
49
+ \t\t{\'c\': 3, \'b\': 2, \'a\': 1}
50
+
51
+ \t\t>>> sort_dict_keys({\'b\': 2, \'a\': 1, \'c\': 3, \'d\': 4}, order=["c", "b"])
52
+ \t\t{\'c\': 3, \'b\': 2, \'a\': 1, \'d\': 4}
53
+ \t'''
54
+ def upsert_in_dataframe(df: pl.DataFrame, new_entry: dict[str, Any], primary_keys: dict[str, Any] | None = None) -> pl.DataFrame:
55
+ """ Insert or update a row in the Polars DataFrame based on primary keys.
56
+
57
+ \tArgs:
58
+ \t\tdf\t\t\t\t(pl.DataFrame):\t\tThe Polars DataFrame to update.
59
+ \t\tnew_entry\t\t(dict[str, Any]):\tThe new entry to insert or update.
60
+ \t\tprimary_keys\t(dict[str, Any]):\tThe primary keys to identify the row (default: empty).
61
+ \tReturns:
62
+ \t\tpl.DataFrame: The updated Polars DataFrame.
63
+ \t"""
64
+ def array_to_disk(data: NDArray[Any] | zarr.Array, delete_input: bool = True, more_data: NDArray[Any] | zarr.Array | None = None) -> tuple['zarr.Array', str, int]:
65
+ """ Easily handle large numpy arrays on disk using zarr for efficient storage and access.
66
+
67
+ \tZarr provides a simpler and more efficient alternative to np.memmap with better compression
68
+ \tand chunking capabilities.
69
+
70
+ \tArgs:
71
+ \t\tdata\t\t\t(NDArray | zarr.Array):\tThe data to save/load as a zarr array
72
+ \t\tdelete_input\t(bool):\tWhether to delete the input data after creating the zarr array
73
+ \t\tmore_data\t\t(NDArray | zarr.Array | None): Additional data to append to the zarr array
74
+ \tReturns:
75
+ \t\ttuple[zarr.Array, str, int]: The zarr array, the directory path, and the total size in bytes
76
+
77
+ \tExamples:
78
+ \t\t>>> import numpy as np
79
+ \t\t>>> data = np.random.rand(1000, 1000)
80
+ \t\t>>> zarr_array = array_to_disk(data)[0]
81
+ \t\t>>> zarr_array.shape
82
+ \t\t(1000, 1000)
83
+
84
+ \t\t>>> more_data = np.random.rand(500, 1000)
85
+ \t\t>>> longer_array, dir_path, total_size = array_to_disk(zarr_array, more_data=more_data)
86
+ \t"""
@@ -0,0 +1,5 @@
1
+ from .cd_utils import *
2
+ from .github import *
3
+ from .pypi import *
4
+ from .pyproject import *
5
+ from .stubs import *
@@ -0,0 +1,129 @@
1
+ import requests
2
+ from ..decorators import handle_error as handle_error
3
+ from ..io import clean_path as clean_path, json_load as json_load
4
+ from ..print import warning as warning
5
+ from typing import Any
6
+
7
+ def load_credentials(credentials_path: str) -> dict[str, Any]:
8
+ ''' Load credentials from a JSON or YAML file into a dictionary.
9
+
10
+ \tLoads credentials from either a JSON or YAML file and returns them as a dictionary.
11
+ \tThe file must contain the required credentials in the appropriate format.
12
+
13
+ \tArgs:
14
+ \t\tcredentials_path (str): Path to the credentials file (.json or .yml)
15
+ \tReturns:
16
+ \t\tdict[str, Any]: Dictionary containing the credentials
17
+
18
+ \tExample JSON format:
19
+
20
+ \t.. code-block:: json
21
+
22
+ \t\t{
23
+ \t\t\t"github": {
24
+ \t\t\t\t"username": "Stoupy51",
25
+ \t\t\t\t"api_key": "ghp_XXXXXXXXXXXXXXXXXXXXXXXXXX"
26
+ \t\t\t}
27
+ \t\t}
28
+
29
+ \tExample YAML format:
30
+
31
+ \t.. code-block:: yaml
32
+
33
+ \t\tgithub:
34
+ \t\t\tusername: "Stoupy51"
35
+ \t\t\tapi_key: "ghp_XXXXXXXXXXXXXXXXXXXXXXXXXX"
36
+ \t'''
37
+ def handle_response(response: requests.Response, error_message: str) -> None:
38
+ """ Handle a response from the API by raising an error if the response is not successful (status code not in 200-299).
39
+
40
+ \tArgs:
41
+ \t\tresponse\t\t(requests.Response): The response from the API
42
+ \t\terror_message\t(str): The error message to raise if the response is not successful
43
+ \t"""
44
+ def clean_version(version: str, keep: str = '') -> str:
45
+ ''' Clean a version string
46
+
47
+ \tArgs:
48
+ \t\tversion\t(str): The version string to clean
49
+ \t\tkeep\t(str): The characters to keep in the version string
50
+ \tReturns:
51
+ \t\tstr: The cleaned version string
52
+
53
+ \t>>> clean_version("v1.e0.zfezf0.1.2.3zefz")
54
+ \t\'1.0.0.1.2.3\'
55
+ \t>>> clean_version("v1.e0.zfezf0.1.2.3zefz", keep="v")
56
+ \t\'v1.0.0.1.2.3\'
57
+ \t>>> clean_version("v1.2.3b", keep="ab")
58
+ \t\'1.2.3b\'
59
+ \t'''
60
+ def version_to_float(version: str, error: bool = True) -> Any:
61
+ ''' Converts a version string into a float for comparison purposes.
62
+ \tThe version string is expected to follow the format of major.minor.patch.something_else....,
63
+ \twhere each part is separated by a dot and can be extended indefinitely.
64
+ \tSupports pre-release suffixes with numbers: devN/dN (dev), aN (alpha), bN (beta), rcN/cN (release candidate).
65
+ \tOrdering: 1.0.0 > 1.0.0rc2 > 1.0.0rc1 > 1.0.0b2 > 1.0.0b1 > 1.0.0a2 > 1.0.0a1 > 1.0.0dev1
66
+
67
+ \tArgs:
68
+ \t\tversion (str): The version string to convert. (e.g. "v1.0.0.1.2.3", "v2.0.0b2", "v1.0.0rc1")
69
+ \t\terror (bool): Return None on error instead of raising an exception
70
+ \tReturns:
71
+ \t\tfloat: The float representation of the version. (e.g. 0)
72
+
73
+ \t>>> version_to_float("v1.0.0")
74
+ \t1.0
75
+ \t>>> version_to_float("v1.0.0.1")
76
+ \t1.000000001
77
+ \t>>> version_to_float("v2.3.7")
78
+ \t2.003007
79
+ \t>>> version_to_float("v1.0.0.1.2.3")
80
+ \t1.0000000010020031
81
+ \t>>> version_to_float("v2.0") > version_to_float("v1.0.0.1")
82
+ \tTrue
83
+ \t>>> version_to_float("v2.0.0") > version_to_float("v2.0.0rc") > version_to_float("v2.0.0b") > version_to_float("v2.0.0a") > version_to_float("v2.0.0dev")
84
+ \tTrue
85
+ \t>>> version_to_float("v1.0.0b") > version_to_float("v1.0.0a")
86
+ \tTrue
87
+ \t>>> version_to_float("v1.0.0") > version_to_float("v1.0.0b")
88
+ \tTrue
89
+ \t>>> version_to_float("v3.0.0a") > version_to_float("v2.9.9")
90
+ \tTrue
91
+ \t>>> version_to_float("v1.2.3b") < version_to_float("v1.2.3")
92
+ \tTrue
93
+ \t>>> version_to_float("1.0.0") == version_to_float("v1.0.0")
94
+ \tTrue
95
+ \t>>> version_to_float("2.0.0.0.0.0.1b") > version_to_float("2.0.0.0.0.0.1a")
96
+ \tTrue
97
+ \t>>> version_to_float("2.0.0.0.0.0.1") > version_to_float("2.0.0.0.0.0.1b")
98
+ \tTrue
99
+ \t>>> version_to_float("v1.0.0rc") == version_to_float("v1.0.0c")
100
+ \tTrue
101
+ \t>>> version_to_float("v1.0.0c") > version_to_float("v1.0.0b")
102
+ \tTrue
103
+ \t>>> version_to_float("v1.0.0d") < version_to_float("v1.0.0a")
104
+ \tTrue
105
+ \t>>> version_to_float("v1.0.0dev") < version_to_float("v1.0.0a")
106
+ \tTrue
107
+ \t>>> version_to_float("v1.0.0dev") == version_to_float("v1.0.0d")
108
+ \tTrue
109
+ \t>>> version_to_float("v1.0.0rc2") > version_to_float("v1.0.0rc1")
110
+ \tTrue
111
+ \t>>> version_to_float("v1.0.0b2") > version_to_float("v1.0.0b1")
112
+ \tTrue
113
+ \t>>> version_to_float("v1.0.0a2") > version_to_float("v1.0.0a1")
114
+ \tTrue
115
+ \t>>> version_to_float("v1.0.0dev2") > version_to_float("v1.0.0dev1")
116
+ \tTrue
117
+ \t>>> version_to_float("v1.0.0") > version_to_float("v1.0.0rc2") > version_to_float("v1.0.0rc1")
118
+ \tTrue
119
+ \t>>> version_to_float("v1.0.0rc1") > version_to_float("v1.0.0b2")
120
+ \tTrue
121
+ \t>>> version_to_float("v1.0.0b1") > version_to_float("v1.0.0a2")
122
+ \tTrue
123
+ \t>>> version_to_float("v1.0.0a1") > version_to_float("v1.0.0dev2")
124
+ \tTrue
125
+ \t>>> versions = ["v1.0.0", "v1.0.0rc2", "v1.0.0rc1", "v1.0.0b2", "v1.0.0b1", "v1.0.0a2", "v1.0.0a1", "v1.0.0dev2", "v1.0.0dev1"]
126
+ \t>>> sorted_versions = sorted(versions, key=version_to_float, reverse=True)
127
+ \t>>> sorted_versions == versions
128
+ \tTrue
129
+ \t'''