stouputils 1.12.1__py3-none-any.whl → 1.13.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stouputils/__init__.py +4 -4
- stouputils/__init__.pyi +1 -0
- stouputils/__main__.py +14 -9
- stouputils/continuous_delivery/pypi.py +39 -1
- stouputils/continuous_delivery/pypi.pyi +9 -0
- stouputils/ctx.py +408 -408
- stouputils/data_science/config/set.py +125 -125
- stouputils/data_science/models/keras_utils/callbacks/model_checkpoint_v2.py +31 -31
- stouputils/data_science/utils.py +285 -285
- stouputils/installer/__init__.py +18 -18
- stouputils/installer/linux.py +144 -144
- stouputils/installer/main.py +223 -223
- stouputils/installer/windows.py +136 -136
- stouputils/print.py +0 -58
- stouputils/print.pyi +0 -10
- stouputils/py.typed +1 -1
- stouputils/stouputils/__init__.pyi +15 -0
- stouputils/stouputils/_deprecated.pyi +12 -0
- stouputils/stouputils/all_doctests.pyi +46 -0
- stouputils/stouputils/applications/__init__.pyi +2 -0
- stouputils/stouputils/applications/automatic_docs.pyi +106 -0
- stouputils/stouputils/applications/upscaler/__init__.pyi +3 -0
- stouputils/stouputils/applications/upscaler/config.pyi +18 -0
- stouputils/stouputils/applications/upscaler/image.pyi +109 -0
- stouputils/stouputils/applications/upscaler/video.pyi +60 -0
- stouputils/stouputils/archive.pyi +67 -0
- stouputils/stouputils/backup.pyi +109 -0
- stouputils/stouputils/collections.pyi +86 -0
- stouputils/stouputils/continuous_delivery/__init__.pyi +5 -0
- stouputils/stouputils/continuous_delivery/cd_utils.pyi +129 -0
- stouputils/stouputils/continuous_delivery/github.pyi +162 -0
- stouputils/stouputils/continuous_delivery/pypi.pyi +53 -0
- stouputils/stouputils/continuous_delivery/pyproject.pyi +67 -0
- stouputils/stouputils/continuous_delivery/stubs.pyi +39 -0
- stouputils/stouputils/ctx.pyi +211 -0
- stouputils/stouputils/decorators.pyi +242 -0
- stouputils/stouputils/image.pyi +172 -0
- stouputils/stouputils/installer/__init__.pyi +5 -0
- stouputils/stouputils/installer/common.pyi +39 -0
- stouputils/stouputils/installer/downloader.pyi +24 -0
- stouputils/stouputils/installer/linux.pyi +39 -0
- stouputils/stouputils/installer/main.pyi +57 -0
- stouputils/stouputils/installer/windows.pyi +31 -0
- stouputils/stouputils/io.pyi +213 -0
- stouputils/stouputils/parallel.pyi +211 -0
- stouputils/stouputils/print.pyi +136 -0
- stouputils/stouputils/version_pkg.pyi +15 -0
- stouputils/version_pkg.py +189 -0
- stouputils/version_pkg.pyi +15 -0
- {stouputils-1.12.1.dist-info → stouputils-1.13.0.dist-info}/METADATA +1 -2
- {stouputils-1.12.1.dist-info → stouputils-1.13.0.dist-info}/RECORD +53 -20
- {stouputils-1.12.1.dist-info → stouputils-1.13.0.dist-info}/WHEEL +0 -0
- {stouputils-1.12.1.dist-info → stouputils-1.13.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from ...installer import check_executable as check_executable
|
|
2
|
+
from ...io import clean_path as clean_path
|
|
3
|
+
from ...parallel import multithreading as multithreading
|
|
4
|
+
from ...print import colored_for_loop as colored_for_loop, debug as debug, info as info
|
|
5
|
+
from .config import Config as Config, WAIFU2X_NCNN_VULKAN_RELEASES as WAIFU2X_NCNN_VULKAN_RELEASES
|
|
6
|
+
from tempfile import TemporaryDirectory
|
|
7
|
+
|
|
8
|
+
def convert_frame(frame_path: str, delete_png: bool = True) -> None:
|
|
9
|
+
''' Convert a PNG frame to JPG format to take less space.
|
|
10
|
+
|
|
11
|
+
\tArgs:
|
|
12
|
+
\t\tframe_path (str): Path to the PNG frame to convert.
|
|
13
|
+
\t\tdelete_png (bool): Whether to delete the original PNG file after conversion.
|
|
14
|
+
|
|
15
|
+
\tReturns:
|
|
16
|
+
\t\tNone: This function doesn\'t return anything.
|
|
17
|
+
|
|
18
|
+
\tExample:
|
|
19
|
+
\t\t.. code-block:: python
|
|
20
|
+
|
|
21
|
+
\t\t\t> convert_frame("input.png", delete_png=True)
|
|
22
|
+
\t\t\t> # input.png will be converted to input.jpg and the original file will be deleted
|
|
23
|
+
|
|
24
|
+
\t\t\t> convert_frame("input.png", delete_png=False)
|
|
25
|
+
\t\t\t> # input.png will be converted to input.jpg and the original file will be kept
|
|
26
|
+
\t'''
|
|
27
|
+
def get_all_files(folder: str, suffix: str | tuple[str, ...] = '') -> list[str]:
|
|
28
|
+
''' Get all files paths in a folder, with a specific suffix if provided.
|
|
29
|
+
|
|
30
|
+
\tArgs:
|
|
31
|
+
\t\tfolder (str): Path to the folder containing the files.
|
|
32
|
+
\t\tsuffix (str | tuple[str, ...]): Suffix of the files to get (e.g. ".png", ".jpg", etc.).
|
|
33
|
+
|
|
34
|
+
\tReturns:
|
|
35
|
+
\t\tlist[str]: List of all files paths in the folder.
|
|
36
|
+
|
|
37
|
+
\tExample:
|
|
38
|
+
\t\t>>> files: list[str] = get_all_files("some_folder", ".png")
|
|
39
|
+
\t\t>>> len(files)
|
|
40
|
+
\t\t0
|
|
41
|
+
\t'''
|
|
42
|
+
def create_temp_dir_for_not_upscaled(input_path: str, output_path: str) -> TemporaryDirectory[str] | None:
|
|
43
|
+
""" Creates a temporary directory containing only images that haven't been upscaled yet.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
input_path (str): Path to the folder containing input images.
|
|
47
|
+
output_path (str): Path to the folder where upscaled images are saved.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
TemporaryDirectory[str] | None: A temporary directory object if there are images to process,
|
|
51
|
+
None if all images are already upscaled.
|
|
52
|
+
"""
|
|
53
|
+
def check_upscaler_executable() -> None: ...
|
|
54
|
+
def upscale(input_path: str, output_path: str, upscale_ratio: int) -> None:
|
|
55
|
+
''' Upscale an input image (or a directory of images) with the upscaler executable.
|
|
56
|
+
|
|
57
|
+
\tArgs:
|
|
58
|
+
\t\tinput_path (str): Path to the image to upscale (or a directory).
|
|
59
|
+
\t\toutput_path (str): Path to the output image (or a directory).
|
|
60
|
+
\t\tupscale_ratio (int): Upscaling ratio.
|
|
61
|
+
|
|
62
|
+
\tExample:
|
|
63
|
+
\t\t.. code-block:: python
|
|
64
|
+
|
|
65
|
+
\t\t\t> upscale("folder", "folder", 2)
|
|
66
|
+
\t\t\tTraceback (most recent call last):
|
|
67
|
+
\t\t\t\t...
|
|
68
|
+
\t\t\tAssertionError: Input and output paths cannot be the same, got \'folder\'
|
|
69
|
+
|
|
70
|
+
\t\t\t> upscale("stouputils", "stouputils/output.jpg", 2)
|
|
71
|
+
\t\t\tTraceback (most recent call last):
|
|
72
|
+
\t\t\t\t...
|
|
73
|
+
\t\t\tAssertionError: If input is a directory, output must be a directory too, got \'stouputils/output.jpg\'
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
\t\t\t> upscale("input.jpg", "output.jpg", 2)
|
|
77
|
+
\t\t\t> # The input.jpg will be upscaled to output.jpg with a ratio of 2
|
|
78
|
+
|
|
79
|
+
\t\t\t> upscale("input_folder", "output_folder", 2)
|
|
80
|
+
\t\t\t> # The input_folder will be upscaled to output_folder with a ratio of 2
|
|
81
|
+
\t'''
|
|
82
|
+
def upscale_images(images: list[str], output_folder: str, upscale_ratio: int, desc: str = 'Upscaling images') -> None:
|
|
83
|
+
""" Upscale multiple images from a list.
|
|
84
|
+
|
|
85
|
+
\tArgs:
|
|
86
|
+
\t\timages (list[str]): List of paths to the images to upscale.
|
|
87
|
+
\t\toutput_folder (str): Path to the output folder where the upscaled images will be saved.
|
|
88
|
+
\t\tupscale_ratio (int): Upscaling ratio.
|
|
89
|
+
\t\tdesc (str): Description of the function execution displayed in the progress bar.
|
|
90
|
+
\t\t\tNo progress bar will be displayed if desc is empty.
|
|
91
|
+
|
|
92
|
+
\tReturns:
|
|
93
|
+
\t\tNone: This function doesn't return anything.
|
|
94
|
+
\t"""
|
|
95
|
+
def upscale_folder(input_folder: str, output_folder: str, upscale_ratio: int, slightly_faster_mode: bool = True, desc: str = 'Upscaling folder') -> None:
|
|
96
|
+
""" Upscale all images in a folder.
|
|
97
|
+
|
|
98
|
+
\tArgs:
|
|
99
|
+
\t\tinput_folder (str): Path to the input folder containing the images to upscale.
|
|
100
|
+
\t\toutput_folder (str): Path to the output folder where the upscaled images will be saved.
|
|
101
|
+
\t\tupscale_ratio (int): Upscaling ratio.
|
|
102
|
+
\t\tslightly_faster_mode (bool): Whether to use the slightly faster mode (no progress bar),
|
|
103
|
+
\t\t\tone call to the upscaler executable.
|
|
104
|
+
\t\tdesc (str): Description of the function execution displayed in the progress bar.
|
|
105
|
+
\t\t\tNo progress bar will be displayed if desc is empty.
|
|
106
|
+
|
|
107
|
+
\tReturns:
|
|
108
|
+
\t\tNone: This function doesn't return anything.
|
|
109
|
+
\t"""
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
from ...installer import check_executable as check_executable
|
|
2
|
+
from ...io import clean_path as clean_path
|
|
3
|
+
from ...parallel import multithreading as multithreading
|
|
4
|
+
from ...print import colored_for_loop as colored_for_loop, debug as debug, error as error, info as info, warning as warning
|
|
5
|
+
from .config import Config as Config, FFMPEG_RELEASES as FFMPEG_RELEASES, YOUTUBE_BITRATE_RECOMMENDATIONS as YOUTUBE_BITRATE_RECOMMENDATIONS
|
|
6
|
+
from .image import convert_frame as convert_frame, get_all_files as get_all_files, upscale_folder as upscale_folder
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
def get_recommended_bitrate(resolution: tuple[int, int], frame_rate: int = 60, upload_type: Literal['SDR', 'HDR'] = 'SDR') -> int:
|
|
10
|
+
''' Get the recommended bitrate (in kbps) for the output video based on the video resolution.
|
|
11
|
+
|
|
12
|
+
\tArgs:
|
|
13
|
+
\t\tresolution (tuple[int, int]): Video resolution (width, height).
|
|
14
|
+
\t\tframe_rate (int): Frame rate of the video, default is 60.
|
|
15
|
+
\t\tupload_type (Literal["SDR","HDR"]): Upload type from which the recommendation is made, default is "SDR".
|
|
16
|
+
|
|
17
|
+
\tReturns:
|
|
18
|
+
\t\tint: The recommended bitrate for the output video (in kbps)
|
|
19
|
+
|
|
20
|
+
\tSource: https://support.google.com/youtube/answer/1722171?hl=en#zippy=%2Cbitrate
|
|
21
|
+
|
|
22
|
+
\tExamples:
|
|
23
|
+
\t\t>>> # Valid examples
|
|
24
|
+
\t\t>>> get_recommended_bitrate((3840, 2160), 60, "SDR")
|
|
25
|
+
\t\t68000
|
|
26
|
+
\t\t>>> get_recommended_bitrate((1920, 1080), 60, "HDR")
|
|
27
|
+
\t\t15000
|
|
28
|
+
\t\t>>> get_recommended_bitrate((1920, 1080), 60, "SDR")
|
|
29
|
+
\t\t12000
|
|
30
|
+
\t\t>>> get_recommended_bitrate((1920, 1080), 30, "SDR")
|
|
31
|
+
\t\t8000
|
|
32
|
+
|
|
33
|
+
\t\t>>> # Invalid examples
|
|
34
|
+
\t\t>>> get_recommended_bitrate((1920, 1080), 60, "Ratio")
|
|
35
|
+
\t\tTraceback (most recent call last):
|
|
36
|
+
\t\t\t...
|
|
37
|
+
\t\tAssertionError: Invalid upload type: \'Ratio\'
|
|
38
|
+
\t\t>>> get_recommended_bitrate("1920x1080", 60, "SDR")
|
|
39
|
+
\t\tTraceback (most recent call last):
|
|
40
|
+
\t\t\t...
|
|
41
|
+
\t\tAssertionError: Invalid resolution: 1920x1080, must be a tuple of two integers
|
|
42
|
+
\t\t>>> get_recommended_bitrate((1920, 1080), -10, "SDR")
|
|
43
|
+
\t\tTraceback (most recent call last):
|
|
44
|
+
\t\t\t...
|
|
45
|
+
\t\tAssertionError: Invalid frame rate: -10, must be a positive integer
|
|
46
|
+
\t'''
|
|
47
|
+
def check_ffmpeg_executable() -> None: ...
|
|
48
|
+
def upscale_video(video_file: str, input_folder: str, progress_folder: str, output_folder: str) -> None:
|
|
49
|
+
""" Handles a video file. """
|
|
50
|
+
def video_upscaler_cli(input_folder: str, progress_folder: str, output_folder: str) -> None:
|
|
51
|
+
""" Upscales videos from an input folder and saves them to an output folder.
|
|
52
|
+
|
|
53
|
+
\tUses intermediate folders for extracted and upscaled frames within the progress folder.
|
|
54
|
+
\t**Handles resuming partially processed videos.**
|
|
55
|
+
|
|
56
|
+
\tArgs:
|
|
57
|
+
\t\tinput_folder (str): Path to the folder containing input videos.
|
|
58
|
+
\t\tprogress_folder (str): Path to the folder for storing intermediate files (frames).
|
|
59
|
+
\t\toutput_folder (str): Path to the folder where upscaled videos will be saved.
|
|
60
|
+
\t"""
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from .decorators import LogLevels as LogLevels, handle_error as handle_error
|
|
2
|
+
from .io import clean_path as clean_path, super_copy as super_copy
|
|
3
|
+
from .print import CYAN as CYAN, GREEN as GREEN, RESET as RESET, debug as debug, error as error, info as info
|
|
4
|
+
|
|
5
|
+
def repair_zip_file(file_path: str, destination: str) -> bool:
|
|
6
|
+
''' Try to repair a corrupted zip file by ignoring some of the errors
|
|
7
|
+
|
|
8
|
+
\tThis function manually parses the ZIP file structure to extract files
|
|
9
|
+
\teven when the ZIP file is corrupted. It reads the central directory
|
|
10
|
+
\tentries and attempts to decompress each file individually.
|
|
11
|
+
|
|
12
|
+
\tArgs:
|
|
13
|
+
\t\tfile_path\t\t(str):\tPath of the zip file to repair
|
|
14
|
+
\t\tdestination\t\t(str):\tDestination of the new file
|
|
15
|
+
\tReturns:
|
|
16
|
+
\t\tbool: Always returns True unless any strong error
|
|
17
|
+
|
|
18
|
+
\tExamples:
|
|
19
|
+
|
|
20
|
+
\t.. code-block:: python
|
|
21
|
+
|
|
22
|
+
\t\t> repair_zip_file("/path/to/source.zip", "/path/to/destination.zip")
|
|
23
|
+
\t'''
|
|
24
|
+
def make_archive(source: str, destinations: list[str] | str | None = None, override_time: None | tuple[int, int, int, int, int, int] = None, create_dir: bool = False, ignore_patterns: str | None = None) -> bool:
|
|
25
|
+
''' Create a zip archive from a source directory with consistent file timestamps.
|
|
26
|
+
\t(Meaning deterministic zip file each time)
|
|
27
|
+
|
|
28
|
+
\tCreates a zip archive from the source directory and copies it to one or more destinations.
|
|
29
|
+
\tThe archive will have consistent file timestamps across runs if override_time is specified.
|
|
30
|
+
\tUses maximum compression level (9) with ZIP_DEFLATED algorithm.
|
|
31
|
+
|
|
32
|
+
\tArgs:
|
|
33
|
+
\t\tsource\t\t\t\t(str):\t\t\t\t\t\tThe source folder to archive
|
|
34
|
+
\t\tdestinations\t\t(list[str]|str):\t\t\tThe destination folder(s) or file(s) to copy the archive to
|
|
35
|
+
\t\toverride_time\t\t(None | tuple[int, ...]):\tThe constant time to use for the archive
|
|
36
|
+
\t\t\t(e.g. (2024, 1, 1, 0, 0, 0) for 2024-01-01 00:00:00)
|
|
37
|
+
\t\tcreate_dir\t\t\t(bool):\t\t\t\t\t\tWhether to create the destination directory if it doesn\'t exist
|
|
38
|
+
\t\tignore_patterns\t\t(str | None):\t\t\t\tGlob pattern(s) to ignore files. Can be a single pattern or comma-separated patterns (e.g. "*.pyc" or "*.pyc,__pycache__,*.log")
|
|
39
|
+
\tReturns:
|
|
40
|
+
\t\tbool: Always returns True unless any strong error
|
|
41
|
+
\tExamples:
|
|
42
|
+
|
|
43
|
+
\t.. code-block:: python
|
|
44
|
+
|
|
45
|
+
\t\t> make_archive("/path/to/source", "/path/to/destination.zip")
|
|
46
|
+
\t\t> make_archive("/path/to/source", ["/path/to/destination.zip", "/path/to/destination2.zip"])
|
|
47
|
+
\t\t> make_archive("src", "hello_from_year_2085.zip", override_time=(2085,1,1,0,0,0))
|
|
48
|
+
\t\t> make_archive("src", "output.zip", ignore_patterns="*.pyc")
|
|
49
|
+
\t\t> make_archive("src", "output.zip", ignore_patterns="__pycache__")
|
|
50
|
+
\t\t> make_archive("src", "output.zip", ignore_patterns="*.pyc,__pycache__,*.log")
|
|
51
|
+
\t'''
|
|
52
|
+
def archive_cli() -> None:
|
|
53
|
+
''' Main entry point for command line usage.
|
|
54
|
+
|
|
55
|
+
\tExamples:
|
|
56
|
+
|
|
57
|
+
\t.. code-block:: bash
|
|
58
|
+
|
|
59
|
+
\t\t# Repair a corrupted zip file
|
|
60
|
+
\t\tpython -m stouputils.archive repair /path/to/corrupted.zip /path/to/repaired.zip
|
|
61
|
+
|
|
62
|
+
\t\t# Create a zip archive
|
|
63
|
+
\t\tpython -m stouputils.archive make /path/to/source /path/to/destination.zip
|
|
64
|
+
|
|
65
|
+
\t\t# Create a zip archive with ignore patterns
|
|
66
|
+
\t\tpython -m stouputils.archive make /path/to/source /path/to/destination.zip --ignore "*.pyc,__pycache__"
|
|
67
|
+
\t'''
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import zipfile
|
|
2
|
+
from .decorators import handle_error as handle_error, measure_time as measure_time
|
|
3
|
+
from .io import clean_path as clean_path
|
|
4
|
+
from .print import CYAN as CYAN, GREEN as GREEN, RESET as RESET, colored_for_loop as colored_for_loop, info as info, warning as warning
|
|
5
|
+
|
|
6
|
+
CHUNK_SIZE: int
|
|
7
|
+
LARGE_CHUNK_SIZE: int
|
|
8
|
+
|
|
9
|
+
def backup_cli() -> None:
|
|
10
|
+
''' Main entry point for command line usage.
|
|
11
|
+
|
|
12
|
+
\tExamples:
|
|
13
|
+
|
|
14
|
+
\t.. code-block:: bash
|
|
15
|
+
|
|
16
|
+
\t\t# Create a delta backup, excluding libraries and cache folders
|
|
17
|
+
\t\tpython -m stouputils.backup delta /path/to/source /path/to/backups -x "libraries/*" "cache/*"
|
|
18
|
+
|
|
19
|
+
\t\t# Consolidate backups into a single file
|
|
20
|
+
\t\tpython -m stouputils.backup consolidate /path/to/backups/latest.zip /path/to/consolidated.zip
|
|
21
|
+
|
|
22
|
+
\t\t# Limit the number of delta backups to 5
|
|
23
|
+
\t\tpython -m stouputils.backup limit 5 /path/to/backups
|
|
24
|
+
\t'''
|
|
25
|
+
def create_delta_backup(source_path: str, destination_folder: str, exclude_patterns: list[str] | None = None) -> None:
|
|
26
|
+
''' Creates a ZIP delta backup, saving only modified or new files while tracking deleted files.
|
|
27
|
+
|
|
28
|
+
\tArgs:
|
|
29
|
+
\t\tsource_path (str): Path to the source file or directory to back up
|
|
30
|
+
\t\tdestination_folder (str): Path to the folder where the backup will be saved
|
|
31
|
+
\t\texclude_patterns (list[str] | None): List of glob patterns to exclude from backup
|
|
32
|
+
\tExamples:
|
|
33
|
+
|
|
34
|
+
\t.. code-block:: python
|
|
35
|
+
|
|
36
|
+
\t\t> create_delta_backup("/path/to/source", "/path/to/backups", exclude_patterns=["libraries/*", "cache/*"])
|
|
37
|
+
\t\t[INFO HH:MM:SS] Creating ZIP backup
|
|
38
|
+
\t\t[INFO HH:MM:SS] Backup created: \'/path/to/backups/backup_2025_02_18-10_00_00.zip\'
|
|
39
|
+
\t'''
|
|
40
|
+
def consolidate_backups(zip_path: str, destination_zip: str) -> None:
|
|
41
|
+
''' Consolidates the files from the given backup and all previous ones into a new ZIP file,
|
|
42
|
+
\tensuring that the most recent version of each file is kept and deleted files are not restored.
|
|
43
|
+
|
|
44
|
+
\tArgs:
|
|
45
|
+
\t\tzip_path (str): Path to the latest backup ZIP file (If endswith "/latest.zip" or "/", the latest backup will be used)
|
|
46
|
+
\t\tdestination_zip (str): Path to the destination ZIP file where the consolidated backup will be saved
|
|
47
|
+
\tExamples:
|
|
48
|
+
|
|
49
|
+
\t.. code-block:: python
|
|
50
|
+
|
|
51
|
+
\t\t> consolidate_backups("/path/to/backups/latest.zip", "/path/to/consolidated.zip")
|
|
52
|
+
\t\t[INFO HH:MM:SS] Consolidating backups
|
|
53
|
+
\t\t[INFO HH:MM:SS] Consolidated backup created: \'/path/to/consolidated.zip\'
|
|
54
|
+
\t'''
|
|
55
|
+
def limit_backups(max_backups: int, backup_folder: str, keep_oldest: bool = True) -> None:
|
|
56
|
+
''' Limits the number of delta backups by consolidating the oldest ones.
|
|
57
|
+
|
|
58
|
+
\tIf the number of backups exceeds max_backups, the oldest backups are consolidated
|
|
59
|
+
\tinto a single backup file, then deleted, until the count is within the limit.
|
|
60
|
+
|
|
61
|
+
\tArgs:
|
|
62
|
+
\t\tmax_backups (int): Maximum number of delta backups to keep
|
|
63
|
+
\t\tbackup_folder (str): Path to the folder containing backups
|
|
64
|
+
\t\tkeep_oldest (bool): If True, never delete the oldest backup (default: True)
|
|
65
|
+
\tExamples:
|
|
66
|
+
|
|
67
|
+
\t.. code-block:: python
|
|
68
|
+
|
|
69
|
+
\t\t> limit_backups(5, "/path/to/backups")
|
|
70
|
+
\t\t[INFO HH:MM:SS] Limiting backups
|
|
71
|
+
\t\t[INFO HH:MM:SS] Consolidated 3 oldest backups into \'/path/to/backups/consolidated_YYYY_MM_DD-HH_MM_SS.zip\'
|
|
72
|
+
\t\t[INFO HH:MM:SS] Deleted 3 old backups
|
|
73
|
+
\t'''
|
|
74
|
+
def get_file_hash(file_path: str) -> str | None:
|
|
75
|
+
""" Computes the SHA-256 hash of a file.
|
|
76
|
+
|
|
77
|
+
\tArgs:
|
|
78
|
+
\t\tfile_path (str): Path to the file
|
|
79
|
+
\tReturns:
|
|
80
|
+
\t\tstr | None: SHA-256 hash as a hexadecimal string or None if an error occurs
|
|
81
|
+
\t"""
|
|
82
|
+
def extract_hash_from_zipinfo(zip_info: zipfile.ZipInfo) -> str | None:
|
|
83
|
+
""" Extracts the stored hash from a ZipInfo object's comment.
|
|
84
|
+
|
|
85
|
+
\tArgs:
|
|
86
|
+
\t\tzip_info (zipfile.ZipInfo): The ZipInfo object representing a file in the ZIP
|
|
87
|
+
\tReturns:
|
|
88
|
+
\t\tstr | None: The stored hash if available, otherwise None
|
|
89
|
+
\t"""
|
|
90
|
+
def get_all_previous_backups(backup_folder: str, all_before: str | None = None) -> dict[str, dict[str, str]]:
|
|
91
|
+
''' Retrieves all previous backups in a folder and maps each backup to a dictionary of file paths and their hashes.
|
|
92
|
+
|
|
93
|
+
\tArgs:
|
|
94
|
+
\t\tbackup_folder (str): The folder containing previous backup zip files
|
|
95
|
+
\t\tall_before (str | None): Path to the latest backup ZIP file
|
|
96
|
+
\t\t\t(If endswith "/latest.zip" or "/", the latest backup will be used)
|
|
97
|
+
\tReturns:
|
|
98
|
+
\t\tdict[str, dict[str, str]]: Dictionary mapping backup file paths to dictionaries of {file_path: file_hash}
|
|
99
|
+
\t'''
|
|
100
|
+
def is_file_in_any_previous_backup(file_path: str, file_hash: str, previous_backups: dict[str, dict[str, str]]) -> bool:
|
|
101
|
+
""" Checks if a file with the same hash exists in any previous backup.
|
|
102
|
+
|
|
103
|
+
\tArgs:
|
|
104
|
+
\t\tfile_path (str): The relative path of the file
|
|
105
|
+
\t\tfile_hash (str): The SHA-256 hash of the file
|
|
106
|
+
\t\tprevious_backups (dict[str, dict[str, str]]): Dictionary mapping backup zip paths to their stored file hashes
|
|
107
|
+
\tReturns:
|
|
108
|
+
\t\tbool: True if the file exists unchanged in any previous backup, False otherwise
|
|
109
|
+
\t"""
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
import zarr
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
from numpy.typing import NDArray as NDArray
|
|
5
|
+
from typing import Any, Literal, TypeVar
|
|
6
|
+
|
|
7
|
+
T = TypeVar('T')
|
|
8
|
+
|
|
9
|
+
def unique_list[T](list_to_clean: Iterable[T], method: Literal['id', 'hash', 'str'] = 'str') -> list[T]:
|
|
10
|
+
''' Remove duplicates from the list while keeping the order using ids (default) or hash or str
|
|
11
|
+
|
|
12
|
+
\tArgs:
|
|
13
|
+
\t\tlist_to_clean\t(Iterable[T]):\t\t\t\t\tThe list to clean
|
|
14
|
+
\t\tmethod\t\t\t(Literal["id", "hash", "str"]):\tThe method to use to identify duplicates
|
|
15
|
+
\tReturns:
|
|
16
|
+
\t\tlist[T]: The cleaned list
|
|
17
|
+
|
|
18
|
+
\tExamples:
|
|
19
|
+
\t\t>>> unique_list([1, 2, 3, 2, 1], method="id")
|
|
20
|
+
\t\t[1, 2, 3]
|
|
21
|
+
|
|
22
|
+
\t\t>>> s1 = {1, 2, 3}
|
|
23
|
+
\t\t>>> s2 = {2, 3, 4}
|
|
24
|
+
\t\t>>> s3 = {1, 2, 3}
|
|
25
|
+
\t\t>>> unique_list([s1, s2, s1, s1, s3, s2, s3], method="id")
|
|
26
|
+
\t\t[{1, 2, 3}, {2, 3, 4}, {1, 2, 3}]
|
|
27
|
+
|
|
28
|
+
\t\t>>> s1 = {1, 2, 3}
|
|
29
|
+
\t\t>>> s2 = {2, 3, 4}
|
|
30
|
+
\t\t>>> s3 = {1, 2, 3}
|
|
31
|
+
\t\t>>> unique_list([s1, s2, s1, s1, s3, s2, s3], method="str")
|
|
32
|
+
\t\t[{1, 2, 3}, {2, 3, 4}]
|
|
33
|
+
\t'''
|
|
34
|
+
def sort_dict_keys[T](dictionary: dict[T, Any], order: list[T], reverse: bool = False) -> dict[T, Any]:
|
|
35
|
+
''' Sort dictionary keys using a given order list (reverse optional)
|
|
36
|
+
|
|
37
|
+
\tArgs:
|
|
38
|
+
\t\tdictionary\t(dict[T, Any]):\tThe dictionary to sort
|
|
39
|
+
\t\torder\t\t(list[T]):\t\tThe order list
|
|
40
|
+
\t\treverse\t\t(bool):\t\t\tWhether to sort in reverse order (given to sorted function which behaves differently than order.reverse())
|
|
41
|
+
\tReturns:
|
|
42
|
+
\t\tdict[T, Any]: The sorted dictionary
|
|
43
|
+
|
|
44
|
+
\tExamples:
|
|
45
|
+
\t\t>>> sort_dict_keys({\'b\': 2, \'a\': 1, \'c\': 3}, order=["a", "b", "c"])
|
|
46
|
+
\t\t{\'a\': 1, \'b\': 2, \'c\': 3}
|
|
47
|
+
|
|
48
|
+
\t\t>>> sort_dict_keys({\'b\': 2, \'a\': 1, \'c\': 3}, order=["a", "b", "c"], reverse=True)
|
|
49
|
+
\t\t{\'c\': 3, \'b\': 2, \'a\': 1}
|
|
50
|
+
|
|
51
|
+
\t\t>>> sort_dict_keys({\'b\': 2, \'a\': 1, \'c\': 3, \'d\': 4}, order=["c", "b"])
|
|
52
|
+
\t\t{\'c\': 3, \'b\': 2, \'a\': 1, \'d\': 4}
|
|
53
|
+
\t'''
|
|
54
|
+
def upsert_in_dataframe(df: pl.DataFrame, new_entry: dict[str, Any], primary_keys: dict[str, Any] | None = None) -> pl.DataFrame:
|
|
55
|
+
""" Insert or update a row in the Polars DataFrame based on primary keys.
|
|
56
|
+
|
|
57
|
+
\tArgs:
|
|
58
|
+
\t\tdf\t\t\t\t(pl.DataFrame):\t\tThe Polars DataFrame to update.
|
|
59
|
+
\t\tnew_entry\t\t(dict[str, Any]):\tThe new entry to insert or update.
|
|
60
|
+
\t\tprimary_keys\t(dict[str, Any]):\tThe primary keys to identify the row (default: empty).
|
|
61
|
+
\tReturns:
|
|
62
|
+
\t\tpl.DataFrame: The updated Polars DataFrame.
|
|
63
|
+
\t"""
|
|
64
|
+
def array_to_disk(data: NDArray[Any] | zarr.Array, delete_input: bool = True, more_data: NDArray[Any] | zarr.Array | None = None) -> tuple['zarr.Array', str, int]:
|
|
65
|
+
""" Easily handle large numpy arrays on disk using zarr for efficient storage and access.
|
|
66
|
+
|
|
67
|
+
\tZarr provides a simpler and more efficient alternative to np.memmap with better compression
|
|
68
|
+
\tand chunking capabilities.
|
|
69
|
+
|
|
70
|
+
\tArgs:
|
|
71
|
+
\t\tdata\t\t\t(NDArray | zarr.Array):\tThe data to save/load as a zarr array
|
|
72
|
+
\t\tdelete_input\t(bool):\tWhether to delete the input data after creating the zarr array
|
|
73
|
+
\t\tmore_data\t\t(NDArray | zarr.Array | None): Additional data to append to the zarr array
|
|
74
|
+
\tReturns:
|
|
75
|
+
\t\ttuple[zarr.Array, str, int]: The zarr array, the directory path, and the total size in bytes
|
|
76
|
+
|
|
77
|
+
\tExamples:
|
|
78
|
+
\t\t>>> import numpy as np
|
|
79
|
+
\t\t>>> data = np.random.rand(1000, 1000)
|
|
80
|
+
\t\t>>> zarr_array = array_to_disk(data)[0]
|
|
81
|
+
\t\t>>> zarr_array.shape
|
|
82
|
+
\t\t(1000, 1000)
|
|
83
|
+
|
|
84
|
+
\t\t>>> more_data = np.random.rand(500, 1000)
|
|
85
|
+
\t\t>>> longer_array, dir_path, total_size = array_to_disk(zarr_array, more_data=more_data)
|
|
86
|
+
\t"""
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from ..decorators import handle_error as handle_error
|
|
3
|
+
from ..io import clean_path as clean_path, json_load as json_load
|
|
4
|
+
from ..print import warning as warning
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
def load_credentials(credentials_path: str) -> dict[str, Any]:
|
|
8
|
+
''' Load credentials from a JSON or YAML file into a dictionary.
|
|
9
|
+
|
|
10
|
+
\tLoads credentials from either a JSON or YAML file and returns them as a dictionary.
|
|
11
|
+
\tThe file must contain the required credentials in the appropriate format.
|
|
12
|
+
|
|
13
|
+
\tArgs:
|
|
14
|
+
\t\tcredentials_path (str): Path to the credentials file (.json or .yml)
|
|
15
|
+
\tReturns:
|
|
16
|
+
\t\tdict[str, Any]: Dictionary containing the credentials
|
|
17
|
+
|
|
18
|
+
\tExample JSON format:
|
|
19
|
+
|
|
20
|
+
\t.. code-block:: json
|
|
21
|
+
|
|
22
|
+
\t\t{
|
|
23
|
+
\t\t\t"github": {
|
|
24
|
+
\t\t\t\t"username": "Stoupy51",
|
|
25
|
+
\t\t\t\t"api_key": "ghp_XXXXXXXXXXXXXXXXXXXXXXXXXX"
|
|
26
|
+
\t\t\t}
|
|
27
|
+
\t\t}
|
|
28
|
+
|
|
29
|
+
\tExample YAML format:
|
|
30
|
+
|
|
31
|
+
\t.. code-block:: yaml
|
|
32
|
+
|
|
33
|
+
\t\tgithub:
|
|
34
|
+
\t\t\tusername: "Stoupy51"
|
|
35
|
+
\t\t\tapi_key: "ghp_XXXXXXXXXXXXXXXXXXXXXXXXXX"
|
|
36
|
+
\t'''
|
|
37
|
+
def handle_response(response: requests.Response, error_message: str) -> None:
|
|
38
|
+
""" Handle a response from the API by raising an error if the response is not successful (status code not in 200-299).
|
|
39
|
+
|
|
40
|
+
\tArgs:
|
|
41
|
+
\t\tresponse\t\t(requests.Response): The response from the API
|
|
42
|
+
\t\terror_message\t(str): The error message to raise if the response is not successful
|
|
43
|
+
\t"""
|
|
44
|
+
def clean_version(version: str, keep: str = '') -> str:
|
|
45
|
+
''' Clean a version string
|
|
46
|
+
|
|
47
|
+
\tArgs:
|
|
48
|
+
\t\tversion\t(str): The version string to clean
|
|
49
|
+
\t\tkeep\t(str): The characters to keep in the version string
|
|
50
|
+
\tReturns:
|
|
51
|
+
\t\tstr: The cleaned version string
|
|
52
|
+
|
|
53
|
+
\t>>> clean_version("v1.e0.zfezf0.1.2.3zefz")
|
|
54
|
+
\t\'1.0.0.1.2.3\'
|
|
55
|
+
\t>>> clean_version("v1.e0.zfezf0.1.2.3zefz", keep="v")
|
|
56
|
+
\t\'v1.0.0.1.2.3\'
|
|
57
|
+
\t>>> clean_version("v1.2.3b", keep="ab")
|
|
58
|
+
\t\'1.2.3b\'
|
|
59
|
+
\t'''
|
|
60
|
+
def version_to_float(version: str, error: bool = True) -> Any:
|
|
61
|
+
''' Converts a version string into a float for comparison purposes.
|
|
62
|
+
\tThe version string is expected to follow the format of major.minor.patch.something_else....,
|
|
63
|
+
\twhere each part is separated by a dot and can be extended indefinitely.
|
|
64
|
+
\tSupports pre-release suffixes with numbers: devN/dN (dev), aN (alpha), bN (beta), rcN/cN (release candidate).
|
|
65
|
+
\tOrdering: 1.0.0 > 1.0.0rc2 > 1.0.0rc1 > 1.0.0b2 > 1.0.0b1 > 1.0.0a2 > 1.0.0a1 > 1.0.0dev1
|
|
66
|
+
|
|
67
|
+
\tArgs:
|
|
68
|
+
\t\tversion (str): The version string to convert. (e.g. "v1.0.0.1.2.3", "v2.0.0b2", "v1.0.0rc1")
|
|
69
|
+
\t\terror (bool): Return None on error instead of raising an exception
|
|
70
|
+
\tReturns:
|
|
71
|
+
\t\tfloat: The float representation of the version. (e.g. 0)
|
|
72
|
+
|
|
73
|
+
\t>>> version_to_float("v1.0.0")
|
|
74
|
+
\t1.0
|
|
75
|
+
\t>>> version_to_float("v1.0.0.1")
|
|
76
|
+
\t1.000000001
|
|
77
|
+
\t>>> version_to_float("v2.3.7")
|
|
78
|
+
\t2.003007
|
|
79
|
+
\t>>> version_to_float("v1.0.0.1.2.3")
|
|
80
|
+
\t1.0000000010020031
|
|
81
|
+
\t>>> version_to_float("v2.0") > version_to_float("v1.0.0.1")
|
|
82
|
+
\tTrue
|
|
83
|
+
\t>>> version_to_float("v2.0.0") > version_to_float("v2.0.0rc") > version_to_float("v2.0.0b") > version_to_float("v2.0.0a") > version_to_float("v2.0.0dev")
|
|
84
|
+
\tTrue
|
|
85
|
+
\t>>> version_to_float("v1.0.0b") > version_to_float("v1.0.0a")
|
|
86
|
+
\tTrue
|
|
87
|
+
\t>>> version_to_float("v1.0.0") > version_to_float("v1.0.0b")
|
|
88
|
+
\tTrue
|
|
89
|
+
\t>>> version_to_float("v3.0.0a") > version_to_float("v2.9.9")
|
|
90
|
+
\tTrue
|
|
91
|
+
\t>>> version_to_float("v1.2.3b") < version_to_float("v1.2.3")
|
|
92
|
+
\tTrue
|
|
93
|
+
\t>>> version_to_float("1.0.0") == version_to_float("v1.0.0")
|
|
94
|
+
\tTrue
|
|
95
|
+
\t>>> version_to_float("2.0.0.0.0.0.1b") > version_to_float("2.0.0.0.0.0.1a")
|
|
96
|
+
\tTrue
|
|
97
|
+
\t>>> version_to_float("2.0.0.0.0.0.1") > version_to_float("2.0.0.0.0.0.1b")
|
|
98
|
+
\tTrue
|
|
99
|
+
\t>>> version_to_float("v1.0.0rc") == version_to_float("v1.0.0c")
|
|
100
|
+
\tTrue
|
|
101
|
+
\t>>> version_to_float("v1.0.0c") > version_to_float("v1.0.0b")
|
|
102
|
+
\tTrue
|
|
103
|
+
\t>>> version_to_float("v1.0.0d") < version_to_float("v1.0.0a")
|
|
104
|
+
\tTrue
|
|
105
|
+
\t>>> version_to_float("v1.0.0dev") < version_to_float("v1.0.0a")
|
|
106
|
+
\tTrue
|
|
107
|
+
\t>>> version_to_float("v1.0.0dev") == version_to_float("v1.0.0d")
|
|
108
|
+
\tTrue
|
|
109
|
+
\t>>> version_to_float("v1.0.0rc2") > version_to_float("v1.0.0rc1")
|
|
110
|
+
\tTrue
|
|
111
|
+
\t>>> version_to_float("v1.0.0b2") > version_to_float("v1.0.0b1")
|
|
112
|
+
\tTrue
|
|
113
|
+
\t>>> version_to_float("v1.0.0a2") > version_to_float("v1.0.0a1")
|
|
114
|
+
\tTrue
|
|
115
|
+
\t>>> version_to_float("v1.0.0dev2") > version_to_float("v1.0.0dev1")
|
|
116
|
+
\tTrue
|
|
117
|
+
\t>>> version_to_float("v1.0.0") > version_to_float("v1.0.0rc2") > version_to_float("v1.0.0rc1")
|
|
118
|
+
\tTrue
|
|
119
|
+
\t>>> version_to_float("v1.0.0rc1") > version_to_float("v1.0.0b2")
|
|
120
|
+
\tTrue
|
|
121
|
+
\t>>> version_to_float("v1.0.0b1") > version_to_float("v1.0.0a2")
|
|
122
|
+
\tTrue
|
|
123
|
+
\t>>> version_to_float("v1.0.0a1") > version_to_float("v1.0.0dev2")
|
|
124
|
+
\tTrue
|
|
125
|
+
\t>>> versions = ["v1.0.0", "v1.0.0rc2", "v1.0.0rc1", "v1.0.0b2", "v1.0.0b1", "v1.0.0a2", "v1.0.0a1", "v1.0.0dev2", "v1.0.0dev1"]
|
|
126
|
+
\t>>> sorted_versions = sorted(versions, key=version_to_float, reverse=True)
|
|
127
|
+
\t>>> sorted_versions == versions
|
|
128
|
+
\tTrue
|
|
129
|
+
\t'''
|