stouputils 1.5.0__tar.gz → 1.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stouputils-1.5.0 → stouputils-1.5.2}/PKG-INFO +3 -3
- {stouputils-1.5.0 → stouputils-1.5.2}/README.md +102 -102
- {stouputils-1.5.0 → stouputils-1.5.2}/pyproject.toml +1 -1
- stouputils-1.5.2/stouputils/__main__.py +58 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/backup.py +118 -4
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/collections.py +26 -1
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/decorators.py +12 -7
- stouputils-1.5.0/stouputils/__main__.py +0 -35
- {stouputils-1.5.0 → stouputils-1.5.2}/.gitignore +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/LICENSE +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/__init__.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/all_doctests.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/applications/__init__.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/applications/automatic_docs.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/applications/upscaler/__init__.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/applications/upscaler/config.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/applications/upscaler/image.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/applications/upscaler/video.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/archive.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/continuous_delivery/__init__.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/continuous_delivery/cd_utils.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/continuous_delivery/github.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/continuous_delivery/pypi.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/continuous_delivery/pyproject.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/ctx.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/config/get.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/config/set.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/__init__.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/auto_contrast.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/axis_flip.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/bias_field_correction.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/binary_threshold.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/blur.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/brightness.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/canny.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/clahe.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/common.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/contrast.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/curvature_flow_filter.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/denoise.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/histogram_equalization.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/invert.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/laplacian.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/median_blur.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/noise.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/normalize.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/random_erase.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/resize.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/rotation.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/salt_pepper.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/sharpening.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/shearing.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/threshold.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/translation.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/zoom.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image_augmentation.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image_preprocess.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/prosthesis_detection.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/technique.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/dataset/__init__.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/dataset/dataset.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/dataset/dataset_loader.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/dataset/grouping_strategy.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/dataset/image_loader.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/dataset/xy_tuple.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/metric_dictionnary.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/metric_utils.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/mlflow_utils.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/abstract_model.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/all.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/base_keras.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras/all.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras/convnext.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras/densenet.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras/efficientnet.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras/mobilenet.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras/resnet.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras/squeezenet.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras/vgg.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras/xception.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/callbacks/__init__.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/callbacks/colored_progress_bar.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/callbacks/learning_rate_finder.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/callbacks/model_checkpoint_v2.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/callbacks/progressive_unfreezing.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/callbacks/warmup_scheduler.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/losses/__init__.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/losses/next_generation_loss.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/visualizations.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/model_interface.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/sandbox.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/range_tuple.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/scripts/augment_dataset.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/scripts/exhaustive_process.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/scripts/preprocess_dataset.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/scripts/routine.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/utils.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/image.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/installer/__init__.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/installer/common.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/installer/downloader.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/installer/linux.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/installer/main.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/installer/windows.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/io.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/parallel.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/print.py +0 -0
- {stouputils-1.5.0 → stouputils-1.5.2}/stouputils/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: stouputils
|
|
3
|
-
Version: 1.5.
|
|
3
|
+
Version: 1.5.2
|
|
4
4
|
Summary: Stouputils is a collection of utility modules designed to simplify and enhance the development process. It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers, and many more.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Stoupy51/stouputils
|
|
6
6
|
Project-URL: Issues, https://github.com/Stoupy51/stouputils/issues
|
|
@@ -117,9 +117,9 @@ It includes a range of tools for tasks such as execution of doctests, display ut
|
|
|
117
117
|
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.all_doctests.html">all_doctests.py</a> <span class="comment"># ✅ Execution of all doctests for a given path</span>
|
|
118
118
|
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.archive.html">archive.py</a> <span class="comment"># 📦 Archive utilities (zip, repair_zip)</span>
|
|
119
119
|
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.backup.html">backup.py</a> <span class="comment"># 📦 Backup utilities (delta backup, consolidate)</span>
|
|
120
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.collections.html">collections.py</a> <span class="comment"># 🧰 Collection utilities (unique_list)</span>
|
|
120
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.collections.html">collections.py</a> <span class="comment"># 🧰 Collection utilities (unique_list, sort_dict_keys, array_to_disk)</span>
|
|
121
121
|
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.ctx.html">ctx.py</a> <span class="comment"># 🚫 Context managers (Muffle, LogToFile)</span>
|
|
122
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.decorators.html">decorators.py</a> <span class="comment"># 🎯 Decorators (silent, measure_time, error_handler, simple_cache)</span>
|
|
122
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.decorators.html">decorators.py</a> <span class="comment"># 🎯 Decorators (silent, measure_time, error_handler, simple_cache, retry)</span>
|
|
123
123
|
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.image.html">image.py</a> <span class="comment"># 🖼️ Image utilities (image_resize)</span>
|
|
124
124
|
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.io.html">io.py</a> <span class="comment"># 💻 I/O utilities (file management, json)</span>
|
|
125
125
|
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.parallel.html">parallel.py</a> <span class="comment"># 🧑🤝🧑 Parallel processing (multiprocessing, multithreading)</span>
|
|
@@ -1,102 +1,102 @@
|
|
|
1
|
-
# 🛠️ Project Badges
|
|
2
|
-
[](https://github.com/Stoupy51/stouputils/releases/latest)
|
|
3
|
-
[](https://pypi.org/project/stouputils/)
|
|
4
|
-
[](https://stoupy51.github.io/stouputils/latest/)
|
|
5
|
-
|
|
6
|
-
<br>
|
|
7
|
-
|
|
8
|
-
# 📚 Project Overview
|
|
9
|
-
Stouputils is a collection of utility modules designed to simplify and enhance the development process.<br>
|
|
10
|
-
It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers.
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
# 🚀 Project File Tree
|
|
14
|
-
<html>
|
|
15
|
-
<details style="display: none;">
|
|
16
|
-
<summary></summary>
|
|
17
|
-
<style>
|
|
18
|
-
.code-tree {
|
|
19
|
-
border-radius: 6px;
|
|
20
|
-
padding: 16px;
|
|
21
|
-
font-family: monospace;
|
|
22
|
-
line-height: 1.45;
|
|
23
|
-
overflow: auto;
|
|
24
|
-
white-space: pre;
|
|
25
|
-
background-color:rgb(43, 43, 43);
|
|
26
|
-
color: #d4d4d4;
|
|
27
|
-
}
|
|
28
|
-
.code-tree a {
|
|
29
|
-
color: #569cd6;
|
|
30
|
-
text-decoration: none;
|
|
31
|
-
}
|
|
32
|
-
.code-tree a:hover {
|
|
33
|
-
text-decoration: underline;
|
|
34
|
-
}
|
|
35
|
-
.code-tree .comment {
|
|
36
|
-
color:rgb(231, 213, 48);
|
|
37
|
-
}
|
|
38
|
-
</style>
|
|
39
|
-
</details>
|
|
40
|
-
|
|
41
|
-
<pre class="code-tree">stouputils/
|
|
42
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.applications.html">applications/</a>
|
|
43
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.applications.automatic_docs.html">automatic_docs.py</a> <span class="comment"># 📚 Documentation generation utilities (used to create this documentation)</span>
|
|
44
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.applications.upscaler.html">upscaler/</a> <span class="comment"># 🔎 Image & Video upscaler (configurable)</span>
|
|
45
|
-
│ └── ...
|
|
46
|
-
│
|
|
47
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.html">continuous_delivery/</a>
|
|
48
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.cd_utils.html">cd_utils.py</a> <span class="comment"># 🔧 Common utilities for continuous delivery</span>
|
|
49
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.github.html">github.py</a> <span class="comment"># 📦 GitHub utilities (upload_to_github)</span>
|
|
50
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.pypi.html">pypi.py</a> <span class="comment"># 📦 PyPI utilities (pypi_full_routine)</span>
|
|
51
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.pyproject.html">pyproject.py</a> <span class="comment"># 📝 Pyproject.toml utilities</span>
|
|
52
|
-
│ └── ...
|
|
53
|
-
│
|
|
54
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.html">data_science/</a>
|
|
55
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.config.html">config/</a> <span class="comment"># ⚙️ Configuration utilities for data science</span>
|
|
56
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.dataset.html">dataset/</a> <span class="comment"># 📊 Dataset handling (dataset, dataset_loader, grouping_strategy)</span>
|
|
57
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.data_processing.html">data_processing/</a> <span class="comment"># 🔄 Data processing utilities (image augmentation, preprocessing)</span>
|
|
58
|
-
│ │ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.data_processing.image.html">image/</a> <span class="comment"># 🖼️ Image processing techniques</span>
|
|
59
|
-
│ │ └── ...
|
|
60
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.models.html">models/</a> <span class="comment"># 🧠 ML/DL model interfaces and implementations</span>
|
|
61
|
-
│ │ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.models.keras.html">keras/</a> <span class="comment"># 🤖 Keras model implementations</span>
|
|
62
|
-
│ │ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.models.keras_utils.html">keras_utils/</a> <span class="comment"># 🛠️ Keras utilities (callbacks, losses, visualizations)</span>
|
|
63
|
-
│ │ └── ...
|
|
64
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.scripts.html">scripts/</a> <span class="comment"># 📜 Data science scripts (augment, preprocess, routine)</span>
|
|
65
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.metric_utils.html">metric_utils.py</a> <span class="comment"># 📏 Metrics utilities for ML/DL models</span>
|
|
66
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.mlflow_utils.html">mlflow_utils.py</a> <span class="comment"># 📊 MLflow integration utilities</span>
|
|
67
|
-
│ └── ...
|
|
68
|
-
│
|
|
69
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.html">installer/</a>
|
|
70
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.common.html">common.py</a> <span class="comment"># 🔧 Common installer utilities</span>
|
|
71
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.downloader.html">downloader.py</a> <span class="comment"># ⬇️ File download utilities</span>
|
|
72
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.linux.html">linux.py</a> <span class="comment"># 🐧 Linux-specific installer utilities</span>
|
|
73
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.main.html">main.py</a> <span class="comment"># 🚀 Main installer functionality</span>
|
|
74
|
-
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.windows.html">windows.py</a> <span class="comment"># 💻 Windows-specific installer utilities</span>
|
|
75
|
-
│ └── ...
|
|
76
|
-
│
|
|
77
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.all_doctests.html">all_doctests.py</a> <span class="comment"># ✅ Execution of all doctests for a given path</span>
|
|
78
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.archive.html">archive.py</a> <span class="comment"># 📦 Archive utilities (zip, repair_zip)</span>
|
|
79
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.backup.html">backup.py</a> <span class="comment"># 📦 Backup utilities (delta backup, consolidate)</span>
|
|
80
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.collections.html">collections.py</a> <span class="comment"># 🧰 Collection utilities (unique_list)</span>
|
|
81
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.ctx.html">ctx.py</a> <span class="comment"># 🚫 Context managers (Muffle, LogToFile)</span>
|
|
82
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.decorators.html">decorators.py</a> <span class="comment"># 🎯 Decorators (silent, measure_time, error_handler, simple_cache)</span>
|
|
83
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.image.html">image.py</a> <span class="comment"># 🖼️ Image utilities (image_resize)</span>
|
|
84
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.io.html">io.py</a> <span class="comment"># 💻 I/O utilities (file management, json)</span>
|
|
85
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.parallel.html">parallel.py</a> <span class="comment"># 🧑🤝🧑 Parallel processing (multiprocessing, multithreading)</span>
|
|
86
|
-
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.print.html">print.py</a> <span class="comment"># 🖨️ Display utilities (info, debug, warning, error)</span>
|
|
87
|
-
└── ...
|
|
88
|
-
</pre>
|
|
89
|
-
</html>
|
|
90
|
-
|
|
91
|
-
## ⭐ Star History
|
|
92
|
-
|
|
93
|
-
<html>
|
|
94
|
-
<a href="https://star-history.com/#Stoupy51/stouputils&Date">
|
|
95
|
-
<picture>
|
|
96
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=Stoupy51/stouputils&type=Date&theme=dark" />
|
|
97
|
-
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=Stoupy51/stouputils&type=Date" />
|
|
98
|
-
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=Stoupy51/stouputils&type=Date" />
|
|
99
|
-
</picture>
|
|
100
|
-
</a>
|
|
101
|
-
</html>
|
|
102
|
-
|
|
1
|
+
# 🛠️ Project Badges
|
|
2
|
+
[](https://github.com/Stoupy51/stouputils/releases/latest)
|
|
3
|
+
[](https://pypi.org/project/stouputils/)
|
|
4
|
+
[](https://stoupy51.github.io/stouputils/latest/)
|
|
5
|
+
|
|
6
|
+
<br>
|
|
7
|
+
|
|
8
|
+
# 📚 Project Overview
|
|
9
|
+
Stouputils is a collection of utility modules designed to simplify and enhance the development process.<br>
|
|
10
|
+
It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers.
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# 🚀 Project File Tree
|
|
14
|
+
<html>
|
|
15
|
+
<details style="display: none;">
|
|
16
|
+
<summary></summary>
|
|
17
|
+
<style>
|
|
18
|
+
.code-tree {
|
|
19
|
+
border-radius: 6px;
|
|
20
|
+
padding: 16px;
|
|
21
|
+
font-family: monospace;
|
|
22
|
+
line-height: 1.45;
|
|
23
|
+
overflow: auto;
|
|
24
|
+
white-space: pre;
|
|
25
|
+
background-color:rgb(43, 43, 43);
|
|
26
|
+
color: #d4d4d4;
|
|
27
|
+
}
|
|
28
|
+
.code-tree a {
|
|
29
|
+
color: #569cd6;
|
|
30
|
+
text-decoration: none;
|
|
31
|
+
}
|
|
32
|
+
.code-tree a:hover {
|
|
33
|
+
text-decoration: underline;
|
|
34
|
+
}
|
|
35
|
+
.code-tree .comment {
|
|
36
|
+
color:rgb(231, 213, 48);
|
|
37
|
+
}
|
|
38
|
+
</style>
|
|
39
|
+
</details>
|
|
40
|
+
|
|
41
|
+
<pre class="code-tree">stouputils/
|
|
42
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.applications.html">applications/</a>
|
|
43
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.applications.automatic_docs.html">automatic_docs.py</a> <span class="comment"># 📚 Documentation generation utilities (used to create this documentation)</span>
|
|
44
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.applications.upscaler.html">upscaler/</a> <span class="comment"># 🔎 Image & Video upscaler (configurable)</span>
|
|
45
|
+
│ └── ...
|
|
46
|
+
│
|
|
47
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.html">continuous_delivery/</a>
|
|
48
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.cd_utils.html">cd_utils.py</a> <span class="comment"># 🔧 Common utilities for continuous delivery</span>
|
|
49
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.github.html">github.py</a> <span class="comment"># 📦 GitHub utilities (upload_to_github)</span>
|
|
50
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.pypi.html">pypi.py</a> <span class="comment"># 📦 PyPI utilities (pypi_full_routine)</span>
|
|
51
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.continuous_delivery.pyproject.html">pyproject.py</a> <span class="comment"># 📝 Pyproject.toml utilities</span>
|
|
52
|
+
│ └── ...
|
|
53
|
+
│
|
|
54
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.html">data_science/</a>
|
|
55
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.config.html">config/</a> <span class="comment"># ⚙️ Configuration utilities for data science</span>
|
|
56
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.dataset.html">dataset/</a> <span class="comment"># 📊 Dataset handling (dataset, dataset_loader, grouping_strategy)</span>
|
|
57
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.data_processing.html">data_processing/</a> <span class="comment"># 🔄 Data processing utilities (image augmentation, preprocessing)</span>
|
|
58
|
+
│ │ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.data_processing.image.html">image/</a> <span class="comment"># 🖼️ Image processing techniques</span>
|
|
59
|
+
│ │ └── ...
|
|
60
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.models.html">models/</a> <span class="comment"># 🧠 ML/DL model interfaces and implementations</span>
|
|
61
|
+
│ │ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.models.keras.html">keras/</a> <span class="comment"># 🤖 Keras model implementations</span>
|
|
62
|
+
│ │ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.models.keras_utils.html">keras_utils/</a> <span class="comment"># 🛠️ Keras utilities (callbacks, losses, visualizations)</span>
|
|
63
|
+
│ │ └── ...
|
|
64
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.scripts.html">scripts/</a> <span class="comment"># 📜 Data science scripts (augment, preprocess, routine)</span>
|
|
65
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.metric_utils.html">metric_utils.py</a> <span class="comment"># 📏 Metrics utilities for ML/DL models</span>
|
|
66
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.data_science.mlflow_utils.html">mlflow_utils.py</a> <span class="comment"># 📊 MLflow integration utilities</span>
|
|
67
|
+
│ └── ...
|
|
68
|
+
│
|
|
69
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.html">installer/</a>
|
|
70
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.common.html">common.py</a> <span class="comment"># 🔧 Common installer utilities</span>
|
|
71
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.downloader.html">downloader.py</a> <span class="comment"># ⬇️ File download utilities</span>
|
|
72
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.linux.html">linux.py</a> <span class="comment"># 🐧 Linux-specific installer utilities</span>
|
|
73
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.main.html">main.py</a> <span class="comment"># 🚀 Main installer functionality</span>
|
|
74
|
+
│ ├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.installer.windows.html">windows.py</a> <span class="comment"># 💻 Windows-specific installer utilities</span>
|
|
75
|
+
│ └── ...
|
|
76
|
+
│
|
|
77
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.all_doctests.html">all_doctests.py</a> <span class="comment"># ✅ Execution of all doctests for a given path</span>
|
|
78
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.archive.html">archive.py</a> <span class="comment"># 📦 Archive utilities (zip, repair_zip)</span>
|
|
79
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.backup.html">backup.py</a> <span class="comment"># 📦 Backup utilities (delta backup, consolidate)</span>
|
|
80
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.collections.html">collections.py</a> <span class="comment"># 🧰 Collection utilities (unique_list, sort_dict_keys, array_to_disk)</span>
|
|
81
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.ctx.html">ctx.py</a> <span class="comment"># 🚫 Context managers (Muffle, LogToFile)</span>
|
|
82
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.decorators.html">decorators.py</a> <span class="comment"># 🎯 Decorators (silent, measure_time, error_handler, simple_cache, retry)</span>
|
|
83
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.image.html">image.py</a> <span class="comment"># 🖼️ Image utilities (image_resize)</span>
|
|
84
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.io.html">io.py</a> <span class="comment"># 💻 I/O utilities (file management, json)</span>
|
|
85
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.parallel.html">parallel.py</a> <span class="comment"># 🧑🤝🧑 Parallel processing (multiprocessing, multithreading)</span>
|
|
86
|
+
├── <a href="https://stoupy51.github.io/stouputils/latest/modules/stouputils.print.html">print.py</a> <span class="comment"># 🖨️ Display utilities (info, debug, warning, error)</span>
|
|
87
|
+
└── ...
|
|
88
|
+
</pre>
|
|
89
|
+
</html>
|
|
90
|
+
|
|
91
|
+
## ⭐ Star History
|
|
92
|
+
|
|
93
|
+
<html>
|
|
94
|
+
<a href="https://star-history.com/#Stoupy51/stouputils&Date">
|
|
95
|
+
<picture>
|
|
96
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=Stoupy51/stouputils&type=Date&theme=dark" />
|
|
97
|
+
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=Stoupy51/stouputils&type=Date" />
|
|
98
|
+
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=Stoupy51/stouputils&type=Date" />
|
|
99
|
+
</picture>
|
|
100
|
+
</a>
|
|
101
|
+
</html>
|
|
102
|
+
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
|
|
6
6
|
[project]
|
|
7
7
|
name = "stouputils"
|
|
8
|
-
version = "1.5.
|
|
8
|
+
version = "1.5.2"
|
|
9
9
|
description = "Stouputils is a collection of utility modules designed to simplify and enhance the development process. It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers, and many more."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.10"
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
# Imports
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
from .all_doctests import launch_tests
|
|
7
|
+
from .backup import backup_cli
|
|
8
|
+
from .decorators import handle_error
|
|
9
|
+
from .print import CYAN, GREEN, RESET, show_version
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@handle_error(message="Error while running 'stouputils'")
|
|
13
|
+
def main():
|
|
14
|
+
second_arg: str = sys.argv[1].lower() if len(sys.argv) >= 2 else ""
|
|
15
|
+
if not second_arg:
|
|
16
|
+
# Get version
|
|
17
|
+
from importlib.metadata import version
|
|
18
|
+
try:
|
|
19
|
+
pkg_version = version("stouputils")
|
|
20
|
+
except Exception:
|
|
21
|
+
pkg_version = "unknown"
|
|
22
|
+
|
|
23
|
+
# Print help with nice formatting
|
|
24
|
+
separator: str = "─" * 60
|
|
25
|
+
print(f"{CYAN}{separator}{RESET}")
|
|
26
|
+
print(f"{CYAN}stouputils {GREEN}CLI {CYAN}v{pkg_version}{RESET}")
|
|
27
|
+
print(f"{CYAN}{separator}{RESET}")
|
|
28
|
+
print(f"\n{CYAN}Usage:{RESET} stouputils <command> [options]")
|
|
29
|
+
print(f"\n{CYAN}Available commands:{RESET}")
|
|
30
|
+
print(f" {GREEN}--version, -v{RESET} Show version information")
|
|
31
|
+
print(f" {GREEN}all_doctests{RESET} [dir] Run all doctests in the specified directory")
|
|
32
|
+
print(f" {GREEN}backup{RESET} --help Backup utilities (delta, consolidate, limit)")
|
|
33
|
+
print(f"{CYAN}{separator}{RESET}")
|
|
34
|
+
return
|
|
35
|
+
|
|
36
|
+
# Print the version of stouputils and its dependencies
|
|
37
|
+
if second_arg in ("--version","-v"):
|
|
38
|
+
return show_version()
|
|
39
|
+
|
|
40
|
+
# Handle "all_doctests" command
|
|
41
|
+
if second_arg == "all_doctests":
|
|
42
|
+
if launch_tests("." if len(sys.argv) == 2 else sys.argv[2]) > 0:
|
|
43
|
+
sys.exit(1)
|
|
44
|
+
return
|
|
45
|
+
|
|
46
|
+
# Handle "backup" command
|
|
47
|
+
if second_arg == "backup":
|
|
48
|
+
sys.argv.pop(1) # Remove "backup" from argv so backup_cli gets clean arguments
|
|
49
|
+
return backup_cli()
|
|
50
|
+
|
|
51
|
+
# Check if the command is any package name
|
|
52
|
+
if second_arg in (): # type: ignore
|
|
53
|
+
return
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
if __name__ == "__main__":
|
|
57
|
+
main()
|
|
58
|
+
|
|
@@ -4,6 +4,7 @@ This module provides utilities for backup management.
|
|
|
4
4
|
- backup_cli: Main entry point for command line usage
|
|
5
5
|
- create_delta_backup: Creates a ZIP delta backup, saving only modified or new files while tracking deleted files
|
|
6
6
|
- consolidate_backups: Consolidates the files from the given backup and all previous ones into a new ZIP file
|
|
7
|
+
- limit_backups: Limits the number of delta backups by consolidating the oldest ones
|
|
7
8
|
- get_file_hash: Computes the SHA-256 hash of a file
|
|
8
9
|
- extract_hash_from_zipinfo: Extracts the stored hash from a ZipInfo object's comment
|
|
9
10
|
- get_all_previous_backups: Retrieves all previous backups in a folder and maps each backup to a dictionary of file paths and their hashes
|
|
@@ -24,7 +25,7 @@ import zipfile
|
|
|
24
25
|
# Local imports
|
|
25
26
|
from .decorators import handle_error, measure_time
|
|
26
27
|
from .io import clean_path
|
|
27
|
-
from .print import colored_for_loop, info,
|
|
28
|
+
from .print import CYAN, GREEN, RESET, colored_for_loop, info, warning
|
|
28
29
|
|
|
29
30
|
# Constants
|
|
30
31
|
CHUNK_SIZE = 1048576 # 1MB chunks for I/O operations
|
|
@@ -32,7 +33,6 @@ LARGE_CHUNK_SIZE = 8388608 # 8MB chunks for large file operations
|
|
|
32
33
|
|
|
33
34
|
|
|
34
35
|
# Main entry point for command line usage
|
|
35
|
-
@measure_time(progress)
|
|
36
36
|
def backup_cli():
|
|
37
37
|
""" Main entry point for command line usage.
|
|
38
38
|
|
|
@@ -45,14 +45,22 @@ def backup_cli():
|
|
|
45
45
|
|
|
46
46
|
# Consolidate backups into a single file
|
|
47
47
|
python -m stouputils.backup consolidate /path/to/backups/latest.zip /path/to/consolidated.zip
|
|
48
|
+
|
|
49
|
+
# Limit the number of delta backups to 5
|
|
50
|
+
python -m stouputils.backup limit 5 /path/to/backups
|
|
48
51
|
"""
|
|
49
52
|
import argparse
|
|
50
53
|
|
|
51
54
|
# Setup command line argument parser
|
|
52
55
|
parser: argparse.ArgumentParser = argparse.ArgumentParser(
|
|
53
|
-
description="Backup and consolidate files using delta compression."
|
|
56
|
+
description="Backup and consolidate files using delta compression.",
|
|
57
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
58
|
+
epilog=f"""{CYAN}Examples:{RESET}
|
|
59
|
+
stouputils backup delta /path/to/source /path/to/backups -x "*.pyc"
|
|
60
|
+
stouputils backup consolidate /path/to/backups/latest.zip /path/to/output.zip
|
|
61
|
+
stouputils backup limit 5 /path/to/backups"""
|
|
54
62
|
)
|
|
55
|
-
subparsers = parser.add_subparsers(dest="command", required=
|
|
63
|
+
subparsers = parser.add_subparsers(dest="command", required=False)
|
|
56
64
|
|
|
57
65
|
# Create delta command and its arguments
|
|
58
66
|
delta_psr = subparsers.add_parser("delta", help="Create a new delta backup")
|
|
@@ -65,13 +73,37 @@ def backup_cli():
|
|
|
65
73
|
consolidate_psr.add_argument("backup_zip", type=str, help="Path to the latest backup ZIP file")
|
|
66
74
|
consolidate_psr.add_argument("destination_zip", type=str, help="Path to the destination consolidated ZIP file")
|
|
67
75
|
|
|
76
|
+
# Create limit command and its arguments
|
|
77
|
+
limit_psr = subparsers.add_parser("limit", help="Limit the number of delta backups by consolidating the oldest ones")
|
|
78
|
+
limit_psr.add_argument("max_backups", type=int, help="Maximum number of delta backups to keep")
|
|
79
|
+
limit_psr.add_argument("backup_folder", type=str, help="Path to the folder containing backups")
|
|
80
|
+
limit_psr.add_argument("--no-keep-oldest", dest="keep_oldest", action="store_false", default=True, help="Allow deletion of the oldest backup (default: keep it)")
|
|
81
|
+
|
|
68
82
|
# Parse arguments and execute appropriate command
|
|
69
83
|
args: argparse.Namespace = parser.parse_args()
|
|
70
84
|
|
|
85
|
+
# Show custom help if no command is provided
|
|
86
|
+
if not args.command:
|
|
87
|
+
separator: str = "─" * 60
|
|
88
|
+
print(f"{CYAN}{separator}{RESET}")
|
|
89
|
+
print(f"{CYAN}Backup Utilities{RESET}")
|
|
90
|
+
print(f"{CYAN}{separator}{RESET}")
|
|
91
|
+
print(f"\n{CYAN}Usage:{RESET} stouputils backup <command> [options]")
|
|
92
|
+
print(f"\n{CYAN}Available commands:{RESET}")
|
|
93
|
+
print(f" {GREEN}delta{RESET} Create a new delta backup")
|
|
94
|
+
print(f" {GREEN}consolidate{RESET} Consolidate existing backups into one")
|
|
95
|
+
print(f" {GREEN}limit{RESET} Limit the number of delta backups")
|
|
96
|
+
print(f"\n{CYAN}For detailed help on a specific command:{RESET}")
|
|
97
|
+
print(" stouputils backup <command> --help")
|
|
98
|
+
print(f"{CYAN}{separator}{RESET}")
|
|
99
|
+
return
|
|
100
|
+
|
|
71
101
|
if args.command == "delta":
|
|
72
102
|
create_delta_backup(args.source, args.destination, args.exclude)
|
|
73
103
|
elif args.command == "consolidate":
|
|
74
104
|
consolidate_backups(args.backup_zip, args.destination_zip)
|
|
105
|
+
elif args.command == "limit":
|
|
106
|
+
limit_backups(args.max_backups, args.backup_folder, keep_oldest=args.keep_oldest)
|
|
75
107
|
|
|
76
108
|
# Main backup function that creates a delta backup (only changed files)
|
|
77
109
|
@measure_time(message="Creating ZIP backup")
|
|
@@ -263,6 +295,10 @@ def consolidate_backups(zip_path: str, destination_zip: str) -> None:
|
|
|
263
295
|
except Exception as e:
|
|
264
296
|
warning(f"Error copying file {filename} from {backup_path}: {e}")
|
|
265
297
|
continue
|
|
298
|
+
|
|
299
|
+
# Add accumulated deleted files to the consolidated backup
|
|
300
|
+
if deleted_files:
|
|
301
|
+
zipf_out.writestr("__deleted_files__.txt", "\n".join(sorted(deleted_files)), compress_type=zipfile.ZIP_DEFLATED)
|
|
266
302
|
finally:
|
|
267
303
|
# Clean up open ZIP files
|
|
268
304
|
for zipf in open_zips.values():
|
|
@@ -273,6 +309,84 @@ def consolidate_backups(zip_path: str, destination_zip: str) -> None:
|
|
|
273
309
|
|
|
274
310
|
info(f"Consolidated backup created: {destination_zip}")
|
|
275
311
|
|
|
312
|
+
# Function to limit the number of delta backups by consolidating the oldest ones
|
|
313
|
+
@measure_time(message="Limiting backups")
|
|
314
|
+
@handle_error()
|
|
315
|
+
def limit_backups(max_backups: int, backup_folder: str, keep_oldest: bool = True) -> None:
|
|
316
|
+
""" Limits the number of delta backups by consolidating the oldest ones.
|
|
317
|
+
|
|
318
|
+
If the number of backups exceeds max_backups, the oldest backups are consolidated
|
|
319
|
+
into a single backup file, then deleted, until the count is within the limit.
|
|
320
|
+
|
|
321
|
+
Args:
|
|
322
|
+
max_backups (int): Maximum number of delta backups to keep
|
|
323
|
+
backup_folder (str): Path to the folder containing backups
|
|
324
|
+
keep_oldest (bool): If True, never delete the oldest backup (default: True)
|
|
325
|
+
Examples:
|
|
326
|
+
|
|
327
|
+
.. code-block:: python
|
|
328
|
+
|
|
329
|
+
> limit_backups(5, "/path/to/backups")
|
|
330
|
+
[INFO HH:MM:SS] Limiting backups
|
|
331
|
+
[INFO HH:MM:SS] Consolidated 3 oldest backups into '/path/to/backups/consolidated_YYYY_MM_DD-HH_MM_SS.zip'
|
|
332
|
+
[INFO HH:MM:SS] Deleted 3 old backups
|
|
333
|
+
"""
|
|
334
|
+
backup_folder = clean_path(os.path.abspath(backup_folder))
|
|
335
|
+
if max_backups < 1:
|
|
336
|
+
raise ValueError("max_backups must be at least 1")
|
|
337
|
+
|
|
338
|
+
# Get all backup files sorted by date (oldest first), including consolidated ones
|
|
339
|
+
# Sort by timestamp (removing "consolidated_" prefix for proper chronological ordering)
|
|
340
|
+
def get_sort_key(filename: str) -> str:
|
|
341
|
+
basename = os.path.basename(filename)
|
|
342
|
+
return basename.replace("consolidated_", "")
|
|
343
|
+
|
|
344
|
+
backup_files: list[str] = sorted([
|
|
345
|
+
clean_path(os.path.join(backup_folder, f))
|
|
346
|
+
for f in os.listdir(backup_folder)
|
|
347
|
+
if f.endswith(".zip")
|
|
348
|
+
], key=get_sort_key)
|
|
349
|
+
|
|
350
|
+
backup_count: int = len(backup_files)
|
|
351
|
+
|
|
352
|
+
# Check if we need to consolidate
|
|
353
|
+
if backup_count <= max_backups:
|
|
354
|
+
info(f"Current backup count ({backup_count}) is within limit ({max_backups}). No action needed.")
|
|
355
|
+
return
|
|
356
|
+
|
|
357
|
+
# Calculate how many backups to consolidate
|
|
358
|
+
num_to_consolidate: int = backup_count - max_backups + 1
|
|
359
|
+
|
|
360
|
+
# If keep_oldest is True, exclude the oldest backup from consolidation
|
|
361
|
+
if keep_oldest and backup_count > 1:
|
|
362
|
+
# Start from index 1 instead of 0 to skip the oldest backup
|
|
363
|
+
backups_to_consolidate: list[str] = backup_files[1:num_to_consolidate+1]
|
|
364
|
+
else:
|
|
365
|
+
backups_to_consolidate: list[str] = backup_files[:num_to_consolidate]
|
|
366
|
+
|
|
367
|
+
latest_to_consolidate: str = backups_to_consolidate[-1]
|
|
368
|
+
|
|
369
|
+
info(f"Found {backup_count} backups, consolidating {num_to_consolidate} oldest backups...")
|
|
370
|
+
|
|
371
|
+
# Extract timestamp from the most recent backup being consolidated (last in list)
|
|
372
|
+
latest_backup: str = os.path.basename(backups_to_consolidate[-1])
|
|
373
|
+
latest_timestamp: str = latest_backup.replace("consolidated_", "").replace(".zip", "")
|
|
374
|
+
|
|
375
|
+
# Create consolidated backup filename with the most recent consolidated backup's timestamp
|
|
376
|
+
consolidated_filename: str = f"consolidated_{latest_timestamp}.zip"
|
|
377
|
+
consolidated_path: str = clean_path(os.path.join(backup_folder, consolidated_filename)) # Consolidate the oldest backups
|
|
378
|
+
consolidate_backups(latest_to_consolidate, consolidated_path)
|
|
379
|
+
|
|
380
|
+
# Delete the old backups that were consolidated
|
|
381
|
+
for backup_path in backups_to_consolidate:
|
|
382
|
+
try:
|
|
383
|
+
os.remove(backup_path)
|
|
384
|
+
info(f"Deleted old backup: {os.path.basename(backup_path)}")
|
|
385
|
+
except Exception as e:
|
|
386
|
+
warning(f"Error deleting backup {backup_path}: {e}")
|
|
387
|
+
|
|
388
|
+
info(f"Successfully limited backups to {max_backups}. Consolidated backup: {consolidated_filename}")
|
|
389
|
+
|
|
276
390
|
# Function to compute the SHA-256 hash of a file
|
|
277
391
|
def get_file_hash(file_path: str) -> str | None:
|
|
278
392
|
""" Computes the SHA-256 hash of a file.
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
This module provides utilities for collection manipulation:
|
|
3
3
|
|
|
4
4
|
- unique_list: Remove duplicates from a list while preserving order using object id, hash or str
|
|
5
|
+
- sort_dict_keys: Sort dictionary keys using a given order list (ascending or descending)
|
|
5
6
|
- array_to_disk: Easily handle large numpy arrays on disk using zarr for efficient storage and access.
|
|
6
7
|
|
|
7
8
|
.. image:: https://raw.githubusercontent.com/Stoupy51/stouputils/refs/heads/main/assets/collections_module.gif
|
|
@@ -13,12 +14,14 @@ import atexit
|
|
|
13
14
|
import os
|
|
14
15
|
import shutil
|
|
15
16
|
import tempfile
|
|
16
|
-
from typing import Any, Literal
|
|
17
|
+
from typing import Any, Literal, TypeVar
|
|
17
18
|
|
|
18
19
|
import numpy as np
|
|
19
20
|
import zarr # pyright: ignore[reportMissingTypeStubs]
|
|
20
21
|
from numpy.typing import NDArray
|
|
21
22
|
|
|
23
|
+
# Typing
|
|
24
|
+
T = TypeVar("T")
|
|
22
25
|
|
|
23
26
|
# Functions
|
|
24
27
|
def unique_list(list_to_clean: list[Any], method: Literal["id", "hash", "str"] = "str") -> list[Any]:
|
|
@@ -69,6 +72,28 @@ def unique_list(list_to_clean: list[Any], method: Literal["id", "hash", "str"] =
|
|
|
69
72
|
# Return the cleaned list
|
|
70
73
|
return result
|
|
71
74
|
|
|
75
|
+
def sort_dict_keys(dictionary: dict[T, Any], order: list[T], reverse: bool = False) -> dict[T, Any]:
|
|
76
|
+
""" Sort dictionary keys using a given order list (reverse optional)
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
dictionary (dict[T, Any]): The dictionary to sort
|
|
80
|
+
order (list[T]): The order list
|
|
81
|
+
reverse (bool): Whether to sort in reverse order (given to sorted function which behaves differently than order.reverse())
|
|
82
|
+
Returns:
|
|
83
|
+
dict[T, Any]: The sorted dictionary
|
|
84
|
+
|
|
85
|
+
Examples:
|
|
86
|
+
>>> sort_dict_keys({'b': 2, 'a': 1, 'c': 3}, order=["a", "b", "c"])
|
|
87
|
+
{'a': 1, 'b': 2, 'c': 3}
|
|
88
|
+
|
|
89
|
+
>>> sort_dict_keys({'b': 2, 'a': 1, 'c': 3}, order=["a", "b", "c"], reverse=True)
|
|
90
|
+
{'c': 3, 'b': 2, 'a': 1}
|
|
91
|
+
|
|
92
|
+
>>> sort_dict_keys({'b': 2, 'a': 1, 'c': 3, 'd': 4}, order=["c", "b"])
|
|
93
|
+
{'c': 3, 'b': 2, 'a': 1, 'd': 4}
|
|
94
|
+
"""
|
|
95
|
+
return dict(sorted(dictionary.items(), key=lambda x: order.index(x[0]) if x[0] in order else len(order), reverse=reverse))
|
|
96
|
+
|
|
72
97
|
def array_to_disk(
|
|
73
98
|
data: NDArray[Any] | zarr.Array,
|
|
74
99
|
delete_input: bool = True,
|
|
@@ -233,16 +233,18 @@ def retry(
|
|
|
233
233
|
exceptions: tuple[type[BaseException], ...] | type[BaseException] = (Exception,),
|
|
234
234
|
max_attempts: int = 10,
|
|
235
235
|
delay: float = 1.0,
|
|
236
|
-
backoff: float = 1.0
|
|
236
|
+
backoff: float = 1.0,
|
|
237
|
+
message: str = ""
|
|
237
238
|
) -> Callable[..., Any]:
|
|
238
239
|
""" Decorator that retries a function when specific exceptions are raised.
|
|
239
240
|
|
|
240
241
|
Args:
|
|
241
|
-
func (Callable[..., Any] | None):
|
|
242
|
-
exceptions (tuple[type[BaseException], ...]):
|
|
243
|
-
max_attempts (int | None):
|
|
244
|
-
delay (float):
|
|
245
|
-
backoff (float):
|
|
242
|
+
func (Callable[..., Any] | None): Function to retry
|
|
243
|
+
exceptions (tuple[type[BaseException], ...]): Exceptions to catch and retry on
|
|
244
|
+
max_attempts (int | None): Maximum number of attempts (None for infinite retries)
|
|
245
|
+
delay (float): Initial delay in seconds between retries (default: 1.0)
|
|
246
|
+
backoff (float): Multiplier for delay after each retry (default: 1.0 for constant delay)
|
|
247
|
+
message (str): Custom message to display before ", retrying" (default: "{ExceptionName} encountered while running {func_name}")
|
|
246
248
|
|
|
247
249
|
Returns:
|
|
248
250
|
Callable[..., Any]: Decorator that retries the function on specified exceptions
|
|
@@ -282,7 +284,10 @@ def retry(
|
|
|
282
284
|
raise e
|
|
283
285
|
|
|
284
286
|
# Log retry attempt
|
|
285
|
-
|
|
287
|
+
if message:
|
|
288
|
+
warning(f"{message}, retrying ({attempt + 1}/{max_attempts}): {e}")
|
|
289
|
+
else:
|
|
290
|
+
warning(f"{type(e).__name__} encountered while running {_get_func_name(func)}, retrying ({attempt + 1}/{max_attempts}): {e}")
|
|
286
291
|
|
|
287
292
|
# Wait before next attempt
|
|
288
293
|
time.sleep(current_delay)
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
# Imports
|
|
4
|
-
import sys
|
|
5
|
-
|
|
6
|
-
from .all_doctests import launch_tests
|
|
7
|
-
from .decorators import handle_error
|
|
8
|
-
from .print import show_version
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
@handle_error(message="Error while running 'stouputils'")
|
|
12
|
-
def main():
|
|
13
|
-
second_arg: str = sys.argv[1].lower() if len(sys.argv) >= 2 else ""
|
|
14
|
-
if not second_arg:
|
|
15
|
-
# TODO
|
|
16
|
-
return
|
|
17
|
-
|
|
18
|
-
# Print the version of stouputils and its dependencies
|
|
19
|
-
if second_arg in ("--version","-v"):
|
|
20
|
-
return show_version()
|
|
21
|
-
|
|
22
|
-
# Handle "all_doctests" command
|
|
23
|
-
if second_arg == "all_doctests":
|
|
24
|
-
if launch_tests("." if len(sys.argv) == 2 else sys.argv[2]) > 0:
|
|
25
|
-
sys.exit(1)
|
|
26
|
-
return
|
|
27
|
-
|
|
28
|
-
# Check if the command is any package name
|
|
29
|
-
if second_arg in (): # type: ignore
|
|
30
|
-
return
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
if __name__ == "__main__":
|
|
34
|
-
main()
|
|
35
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/__init__.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/auto_contrast.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/axis_flip.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/brightness.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/canny.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/clahe.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/common.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/contrast.py
RENAMED
|
File without changes
|
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/denoise.py
RENAMED
|
File without changes
|
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/invert.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/laplacian.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/median_blur.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/noise.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/normalize.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/random_erase.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/resize.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/rotation.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/salt_pepper.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/sharpening.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/shearing.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/threshold.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image/translation.py
RENAMED
|
File without changes
|
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image_augmentation.py
RENAMED
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/data_processing/image_preprocess.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/losses/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{stouputils-1.5.0 → stouputils-1.5.2}/stouputils/data_science/models/keras_utils/visualizations.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|